Repository: nsqio/nsq Branch: master Commit: 9ea507128926 Files: 211 Total size: 1.2 MB Directory structure: gitextract_w7hjwstt/ ├── .github/ │ └── workflows/ │ └── test.yml ├── .gitignore ├── AUTHORS ├── CODE_OF_CONDUCT.md ├── CONTRIBUTING.md ├── ChangeLog.md ├── Dockerfile ├── LICENSE ├── Makefile ├── README.md ├── apps/ │ ├── nsq_stat/ │ │ └── nsq_stat.go │ ├── nsq_tail/ │ │ └── nsq_tail.go │ ├── nsq_to_file/ │ │ ├── file_logger.go │ │ ├── nsq_to_file.go │ │ ├── options.go │ │ ├── strftime.go │ │ └── topic_discoverer.go │ ├── nsq_to_http/ │ │ ├── http.go │ │ ├── nsq_to_http.go │ │ └── nsq_to_http_test.go │ ├── nsq_to_nsq/ │ │ └── nsq_to_nsq.go │ ├── nsqadmin/ │ │ ├── main.go │ │ ├── main_test.go │ │ └── options.go │ ├── nsqd/ │ │ ├── README.md │ │ ├── main.go │ │ ├── main_test.go │ │ └── options.go │ ├── nsqlookupd/ │ │ ├── README.md │ │ ├── main.go │ │ ├── main_test.go │ │ └── options.go │ └── to_nsq/ │ ├── README.md │ └── to_nsq.go ├── bench/ │ ├── bench.py │ ├── bench_channels/ │ │ └── bench_channels.go │ ├── bench_reader/ │ │ └── bench_reader.go │ ├── bench_writer/ │ │ └── bench_writer.go │ └── requirements.txt ├── bench.sh ├── contrib/ │ ├── nsq.spec │ ├── nsqadmin.cfg.example │ ├── nsqd.cfg.example │ └── nsqlookupd.cfg.example ├── coverage.sh ├── dist.sh ├── fmt.sh ├── go.mod ├── go.sum ├── internal/ │ ├── app/ │ │ ├── float_array.go │ │ └── string_array.go │ ├── auth/ │ │ └── authorizations.go │ ├── clusterinfo/ │ │ ├── data.go │ │ ├── producer_test.go │ │ └── types.go │ ├── dirlock/ │ │ ├── dirlock.go │ │ ├── dirlock_illumos.go │ │ └── dirlock_windows.go │ ├── http_api/ │ │ ├── api_request.go │ │ ├── api_response.go │ │ ├── compress.go │ │ ├── http_server.go │ │ ├── req_params.go │ │ └── topic_channel_args.go │ ├── lg/ │ │ ├── lg.go │ │ └── lg_test.go │ ├── pqueue/ │ │ ├── pqueue.go │ │ └── pqueue_test.go │ ├── protocol/ │ │ ├── byte_base10.go │ │ ├── byte_base10_test.go │ │ ├── errors.go │ │ ├── names.go │ │ ├── protocol.go │ │ └── tcp_server.go │ ├── quantile/ │ │ ├── aggregate.go │ │ └── quantile.go │ ├── statsd/ │ │ ├── client.go │ │ └── host.go │ ├── stringy/ │ │ ├── slice.go │ │ ├── slice_test.go │ │ └── template.go │ ├── test/ │ │ ├── assertions.go │ │ ├── fakes.go │ │ └── logger.go │ ├── util/ │ │ ├── rand.go │ │ ├── unix_socket.go │ │ ├── util_test.go │ │ └── wait_group_wrapper.go │ ├── version/ │ │ └── binary.go │ └── writers/ │ ├── boundary_buffered_writer.go │ └── spread_writer.go ├── nsqadmin/ │ ├── .eslintrc │ ├── README.md │ ├── gulp │ ├── gulpfile.js │ ├── http.go │ ├── http_test.go │ ├── logger.go │ ├── notify.go │ ├── nsqadmin.go │ ├── nsqadmin_test.go │ ├── options.go │ ├── package.json │ ├── static/ │ │ ├── build/ │ │ │ ├── base.css │ │ │ ├── index.html │ │ │ ├── main.js │ │ │ └── vendor.js │ │ ├── css/ │ │ │ └── base.scss │ │ ├── html/ │ │ │ └── index.html │ │ └── js/ │ │ ├── app_state.js │ │ ├── collections/ │ │ │ ├── nodes.js │ │ │ └── topics.js │ │ ├── lib/ │ │ │ ├── ajax_setup.js │ │ │ ├── handlebars_helpers.js │ │ │ └── pubsub.js │ │ ├── main.js │ │ ├── models/ │ │ │ ├── channel.js │ │ │ ├── node.js │ │ │ └── topic.js │ │ ├── router.js │ │ └── views/ │ │ ├── app.js │ │ ├── base.js │ │ ├── channel.hbs │ │ ├── channel.js │ │ ├── counter.hbs │ │ ├── counter.js │ │ ├── error.hbs │ │ ├── header.hbs │ │ ├── header.js │ │ ├── lookup.hbs │ │ ├── lookup.js │ │ ├── node.hbs │ │ ├── node.js │ │ ├── nodes.hbs │ │ ├── nodes.js │ │ ├── spinner.hbs │ │ ├── topic.hbs │ │ ├── topic.js │ │ ├── topics.hbs │ │ ├── topics.js │ │ └── warning.hbs │ ├── static.go │ └── test/ │ ├── ca.key │ ├── ca.pem │ ├── ca.srl │ ├── cert.pem │ ├── client.key │ ├── client.pem │ ├── client.req │ ├── key.pem │ ├── server.key │ ├── server.pem │ └── server.req ├── nsqd/ │ ├── README.md │ ├── backend_queue.go │ ├── buffer_pool.go │ ├── channel.go │ ├── channel_test.go │ ├── client_v2.go │ ├── dqname.go │ ├── dqname_windows.go │ ├── dummy_backend_queue.go │ ├── guid.go │ ├── guid_test.go │ ├── http.go │ ├── http_test.go │ ├── in_flight_pqueue.go │ ├── in_flight_pqueue_test.go │ ├── logger.go │ ├── lookup.go │ ├── lookup_peer.go │ ├── message.go │ ├── nsqd.go │ ├── nsqd_test.go │ ├── options.go │ ├── protocol_v2.go │ ├── protocol_v2_test.go │ ├── protocol_v2_unixsocket_test.go │ ├── stats.go │ ├── stats_test.go │ ├── statsd.go │ ├── tcp.go │ ├── test/ │ │ ├── cert.sh │ │ ├── certs/ │ │ │ ├── ca.key │ │ │ ├── ca.pem │ │ │ ├── ca.srl │ │ │ ├── cert.pem │ │ │ ├── client.key │ │ │ ├── client.pem │ │ │ ├── client.req │ │ │ ├── key.pem │ │ │ ├── server.key │ │ │ ├── server.pem │ │ │ └── server.req │ │ └── openssl.conf │ ├── topic.go │ └── topic_test.go ├── nsqlookupd/ │ ├── README.md │ ├── client_v1.go │ ├── http.go │ ├── http_test.go │ ├── logger.go │ ├── lookup_protocol_v1.go │ ├── lookup_protocol_v1_test.go │ ├── nsqlookupd.go │ ├── nsqlookupd_test.go │ ├── options.go │ ├── registration_db.go │ ├── registration_db_test.go │ └── tcp.go └── test.sh ================================================ FILE CONTENTS ================================================ ================================================ FILE: .github/workflows/test.yml ================================================ name: tests on: push: {branches: [master]} pull_request: {branches: [master]} jobs: test: runs-on: ubuntu-20.04 timeout-minutes: 30 strategy: fail-fast: false matrix: go: ["1.21.x", "1.22.x", "1.23.x"] arch: ["amd64", "386"] env: GOARCH: "${{matrix.arch}}" steps: - uses: actions/checkout@v2 - uses: WillAbides/setup-go-faster@v1.7.0 with: go-version: ${{matrix.go}} - name: build run: make all - name: test run: ./test.sh staticcheck: runs-on: ubuntu-20.04 steps: - uses: actions/checkout@v2 - uses: dominikh/staticcheck-action@v1.3.1 with: version: "2024.1.1" install-go: false code-coverage: runs-on: ubuntu-20.04 steps: - uses: actions/checkout@v2 - name: install goveralls run: go install github.com/mattn/goveralls@latest - name: send coverage env: COVERALLS_TOKEN: ${{secrets.GITHUB_TOKEN}} run: ./coverage.sh --coveralls ================================================ FILE: .gitignore ================================================ /build/ dist/ .cover/ profile/ # nsqd data from testing *.dat # nsqadmin node_modules # apps apps/nsqlookupd/nsqlookupd apps/nsqd/nsqd apps/nsqadmin/nsqadmin apps/nsq_to_nsq/nsq_to_nsq apps/nsq_to_file/nsq_to_file apps/nsq_pubsub/nsq_pubsub apps/nsq_to_http/nsq_to_http apps/nsq_tail/nsq_tail apps/nsq_stat/nsq_stat apps/to_nsq/to_nsq bench/bench_reader/bench_reader bench/bench_writer/bench_writer bench/bench_channels/bench_channels # Go.gitignore # Compiled Object files, Static and Dynamic libs (Shared Objects) *.o *.a *.so # Folders _obj _test # Architecture specific extensions/prefixes *.[568vq] [568vq].out *.cgo1.go *.cgo2.c _cgo_defun.c _cgo_gotypes.go _cgo_export.* _testmain.go *.exe # vim stuff *.sw[op] ================================================ FILE: AUTHORS ================================================ # For a complete listing, see https://github.com/nsqio/nsq/graphs/contributors # Original Authors Matt Reiferson Jehiah Czebotar # Maintainers Pierce Lopez # Disclaimer Matt Reiferson's contributions to this project are being made solely in a personal capacity and does not convey any rights to any intellectual property of any third parties. ================================================ FILE: CODE_OF_CONDUCT.md ================================================ # Contributor Code of Conduct As contributors and maintainers of this project, and in the interest of fostering an open and welcoming community, we pledge to respect all people who contribute through reporting issues, posting feature requests, updating documentation, submitting pull requests or patches, and other activities. We are committed to making participation in this project a harassment-free experience for everyone, regardless of level of experience, gender, gender identity and expression, sexual orientation, disability, personal appearance, body size, race, ethnicity, age, religion, or nationality. Examples of unacceptable behavior by participants include: * The use of sexualized language or imagery * Personal attacks * Trolling or insulting/derogatory comments * Public or private harassment * Publishing other's private information, such as physical or electronic addresses, without explicit permission * Other unethical or unprofessional conduct. Project maintainers have the right and responsibility to remove, edit, or reject comments, commits, code, wiki edits, issues, and other contributions that are not aligned to this Code of Conduct. By adopting this Code of Conduct, project maintainers commit themselves to fairly and consistently applying these principles to every aspect of managing this project. Project maintainers who do not follow or enforce the Code of Conduct may be permanently removed from the project team. This code of conduct applies both within project spaces and in public spaces when an individual is representing the project or its community. Instances of abusive, harassing, or otherwise unacceptable behavior may be reported by opening an issue or contacting one or more of the project maintainers. This Code of Conduct is adapted from the [Contributor Covenant][1], version 1.2.0, available at [http://contributor-covenant.org/version/1/2/0/][2]. [1]: http://contributor-covenant.org [2]: http://contributor-covenant.org/version/1/2/0/ ================================================ FILE: CONTRIBUTING.md ================================================ # Contributing Thanks for your interest in contributing to NSQ! ## Code of Conduct Help us keep NSQ open and inclusive. Please read and follow our [Code of Conduct](CODE_OF_CONDUCT.md). ## Getting Started * make sure you have a [GitHub account](https://github.com/signup/free) * submit a ticket for your issue, assuming one does not already exist * clearly describe the issue including steps to reproduce when it is a bug * identify specific versions of the binaries and client libraries * fork the repository on GitHub ## Making Changes * create a branch from where you want to base your work * we typically name branches according to the following format: `helpful_name_` * make commits of logical units * make sure your commit messages are in a clear and readable format, example: ``` nsqd: fixed bug in protocol_v2 * update the message pump to properly account for RDYness * cleanup variable names * ... ``` * if you're fixing a bug or adding functionality it probably makes sense to write a test * make sure to run `fmt.sh` and `test.sh` in the root of the repo to ensure that your code is properly formatted and that tests pass (we use GitHub Actions for continuous integration) ## Submitting Changes * push your changes to your branch in your fork of the repository * submit a pull request against nsqio's repository * comment in the pull request when you're ready for the changes to be reviewed: `"ready for review"` ================================================ FILE: ChangeLog.md ================================================ # NSQ Changelog ## Releases ### 1.3.0 - 2023-12-26 **Upgrading** * #1427 / #1373 / #1371 - fix staticcheck warnings, remove support for gobindata / go 1.16 Features: * #1473 - `nsqd`: use --tls-root-ca-file in nsqauth request (thanks @intellitrend-team) * #1470 / #1469 - `nsqadmin`: upgrade supported ECMA from ES5 to ES2020 (thanks @dudleycarr) * #1468 - `nsqadmin`: add paused label to topic within the node view (thanks @dudleycarr) * #1462 - `nsqadmin`: add admin check for topic/node thombstone endpoint (thanks @dudleycarr) * #1434 - `nsqd`: add support of unix sockets for tcp, http, https listeners (thanks @telepenin) * #1424 - `nsqd`: add /debug/freememory API (thanks @guozhao-coder) * #1421 - `nsqd`: nicer tls-min-version help text default * #1376 - `nsqd`: allow unbuffered memory chan if ephemeral or deferred * #1380 - `nsqd`: use metadata struct for both marshal and unmarshal (thanks @karalabe) * #1403 - `nsqd`: /info api returns more info (thanks @arshabbir) * #1384 - `nsqd`: allow disabling both HTTP and HTTPS interfaces (thanks @karalabe) * #1385 - `nsqd`: enable support for TLS1.3 (thanks @karalabe) * #1372 - `nsqadmin`: new flag --dev-static-dir instead of debug build tag Bugs: * #1478 - `Dockerfile`: remove nsswitch.conf check (thanks @dudleycarr) * #1467 - `nsqadmin`: fix counter by bounding animation steps (thanks @dudleycarr) * #1466 - `nsqadmin`: fix broken graph template in nsqadmin node view (thanks @dudleycarr) * #1455 / #1387 - update dependencies * #1445 - `nsqd`: fix unsafe concurrency read in RemoveClient (thanks @gueFDF) * #1441 - `nsqd`: fix panic when statsd enabled and memstats disabled with no topics (thanks @carl-reverb) * #1428 - delete `validTopicChannelNameRegex` useless escape characters (thanks @sjatsh) * #1419 - contrib: update nsqadmin.cfg.example (thanks @StellarisW) ### 1.2.1 - 2021-08-15 **Upgrading** * #1227 - bump dependencies, officially drop `dep` support, drop Go `1.9` support Features: * #1347 - `nsqadmin`: switch to go:embed for static assets * #1355 / #1364 - arm64 builds (thanks @danbf) * #1346 - `nsqd`: ability to skip ephemeral topics/channels in statsd output * #1336 / #1341 / #1343 - `nsqd`: ability to configure auth endpoint path (thanks @tufanbarisyildirim) * #1307 - remove `Context` to use stdlib `context` * #1295 / #1296 - switch to GitHub Actions CI * #1292 - `nsqd`: minimize allocations on message send (thanks @imxyb) * #1289 - optimize `uniq` (thanks @donutloop) * #1230 / #1232 - `nsqd`: ability to omit memory stats from `/stats` (thanks @creker) * #1226 - `nsqd`: only update internal `RDY` count for client when it changes (thanks @andyxning) * #1221 / #1363 - test against more recent versions of Go * #1209 - `nsqd`: bump `go-diskqueue` (interface change) (thanks @bitpeng) * #1206 - prefer idiomatic `sort.Ints` over `sort.Sort` (thanks @lelenanam) * #1197 / #1362 - Dockerfile: update Alpine base image, use /data by default * #1178 - `nsqd`: configurable queue scan worker pool (thanks @andyxning) * #1159 - `nsqd`: don't buffer messages when `--mem-queue-size=0` (thanks @bitpeng) * #1073 / #1297 - `nsqd`: support separate broadcast ports for TCP and HTTP (thanks @shyam-king) Bugs: * #1347 - `nsqadmin`: fix graphite key for ephemeral topics/channels * #765 / #1195 / #1203 / #1205 - fix build on illumos (thanks @i-sevostyanov) * #1333 - fix race detector tests on non-bash shells * #1330 - fix `log_level` support in configuration file (thanks @edoger) * #1319 / #1331 / #1361 - `nsqd`: handle SIGTERM * #1287 - `nsqadmin`: fix `--proxy-graphite` support (thanks @fanlix) * #1270 / #1271 - `nsqlookupd`: fix incorrect error message for HTTP listener (thanks @TangDH03) * #1264 - fix benchmark script * #1251 / #1314 / #1327 - `nsqd`: fix live lock for high churn ephemeral topic/channel reconnections (thanks @slayercat) * #1237 - Dockerfile: add `nsswitch.conf` to ensure go resolver uses `/etc/hosts` first * #1217 / #1220 - `nsqd`: improve error message when `--data-path` does not exist (thanks @mdh67899) * #1198 / #1190 / #1262 - synchronize close of all connections on Exit (thanks @benjsto) * #1188 / #1189 - `nsqadmin`: fix channel delete, fix source-maps in Firefox (thanks @avtrifanov) * #1186 - `nsqadmin`: fix nodes list with ipv6 addresses (thanks @andyxning) ### 1.2.0 - 2019-08-26 **Upgrading** * #1055 - `nsqd`: removed support for old metadata scheme used in v0.3.8 and earlier * you cannot upgrade directly from v0.3.8 to v1.2.0, you must go through v1.0.0-compat or v1.1.0 * #1115 - manage dependencies with go modules * `dep` support still present for now, but deprecated Features: * #1136 - `nsqd`: add `--max-channel-consumers` (default unlimited) (thanks @andyxning) * #1133 - `nsqd`: add `--min-output-buffer-timeout` (default 25ms) to limit how low a timeout a consumer can request * and raise default `--max-output-buffer-timeout` to 30 seconds (lower timeout, more cpu usage) * #1127 - `nsqd`: add topic total message bytes to stats (thanks @andyxning) * #1125 - `nsqd`: add flag to adjust default `--output-buffer-timeout` (thanks @andyxning) * #1163 - `nsqd`: add random load balancing for authd requests (thanks @shenhui0509) * #1119 - `nsqd`: include client TLS cert CommonName in authd requests * #1147 - `nsq_to_file`: include topic/channel in most log messages * #1117 - `nsq_to_file`: add `--log-level` and `--log-prefix` flags * #1117/#1120/#1123 - `nsq_to_file`: big refactor, more robust file switching and syncing and error handling * #1118 - `nsqd`: add param to `/stats` endpoint to allow skipping per-client stats (much faster if many clients) * #1118 - `nsqadmin`, `nsq_stat`: use `include_clients` param for `/stats` for a big speedup for big clusters * #1110 - `nsq_to_file`: support for separate working directory with `--work-dir` (thanks @mccutchen) * #856 - `nsqadmin`: add `--base-path` flag (thanks @blinklv) * #1072 - `nsq_to_http`: add `--header` flag (thanks @alwindoss) * #881 - `nsqd`: add producer client tcp connections to stats (thanks @sparklxb) * #1071/#1074 - `nsq_to_file`: new flag `--sync-interval` (default same as previous behavior, 30 seconds) (thanks @alpaker) Bugs: * #1153 - `nsqd`: close connections that don't send "magic" header (thanks @JoseFeng) * #1140 - `nsqd`: exit on all fatal Accept() errors - restart enables better recovery for some conditions (thanks @mdh67899) * #1140 - `nsqd`, `nsqlookupd`, `nsqadmin`: refactor LogLevel, general refactor to better exit on all fatal errors * #1140 - `nsqadmin`: switch to using `judwhite/go-svc` like `nsqd` and `nsqadmin` do * #1134 - `nsqadmin`: fix clients count and channel total message rate (new bugs introduced in this cycle) * #1132 - `nsqd`, `nsqlookupd`, `nsqadmin`: fix http error response unreliable json serialization * #1116 - `nsqlookupd`: fix orphaned ephemeral topics in registration DB * #1109 - `nsqd`: fix topic message mis-counting if there are backend write errors (thanks @SwanSpouse) * #1099 - `nsqlookupd`: optimize `/nodes` endpoint, much better for hundreds of nsqd (thanks @andyxning) * #1085 - switch `GOFLAGS` to `BLDFLAGS` in `Makefile` now that `GOFLAGS` is automatically used by go * #1080 - `nsqadmin`: eslint reported fixes/cleanups ### 1.1.0 - 2018-08-19 **Upgrading from 1.0.0-compat**: Just a few backwards incompatible changes: * #1056 - Removed the `nsq_pubsub` utility * #873 - `nsqd` flags `--msg-timeout` and `--statsd-interval` only take duration strings * plain integer no longer supported * #921 - `nsqd`: http `/mpub` endpoint `binary` param interprets "0" or "false" to mean text mode * previously any value meant to use binary mode instead of text mode - (thanks @andyxning) The previous release, version "1.0.0-compat", was curiously-named to indicate an almost (but not quite) complete transition to a 1.0 api-stable release line. Confusingly, this follow-up release which completes the transition comes more than a year later. Because there have been a fair number of changes and improvements in the past year, an additional minor version bump seems appropriate. Features: * #874 - `nsqd`: add memory stats to http `/stats` response (thanks @sparklxb) * #892 - `nsqd`, `nsqlookupd`, `nsqadmin`: add `--log-level` option (deprecating `--verbose`) (thanks @antihax) * #898 - `nsqd`, `nsqlookupd`, `nsqadmin`: logging refactor to use log levels everywhere * #914 - `nsqadmin`: `X-Forwarded-User` based "admin" permission (thanks @chen-anders) * #929 - `nsqd`: add topic/channel filter to `/stats`, use in `nsqadmin` and `nsq_stat` for efficiency (thanks @andyxning) * #936 - `nsq_to_file`: refactor/cleanup * #945 - `nsq_to_nsq`: support multiple `--topic` flags (thanks @jlr52) * #957 - `nsq_tail`: support multiple `--topic` flags (thanks @soar) * #946 - `nsqd`, `nsqadmin`: update internal http client with new go `http.Transport` features (keepalives, timeouts, dualstack) * affects metadata/stats requests between `nsqadmin`, `nsqd`, `nsqlookupd` * #954 - manage dependencies with `dep` (replacing `gpm`) (thanks @judwhite) * #957 - multi-stage docker image build (thanks @soar) * #996 - `nsqd`: better memory usage when messages have different sizes (thanks @andyxning) * #1019 - `nsqd`: optimize random channel selection in queueScanLoop (thanks @vearne) * #1025 - `nsqd`: buffer and spread statsd udp sends (avoid big burst of udp, less chance of loss) * #1038 - `nsqlookupd`: optimize for many producers (thousands) (thanks @andyxning) * #1050/#1053 - `nsqd`: new topic can be unlocked faster after creation * #1062 - `nsqadmin`: update JS deps Bugs: * #753 - `nsqadmin`: fix missing channels in topic list * #867 - `to_nsq`: fix divide-by-zero issue when `--rate` not specified (thanks @adamweiner) * #868 - `nsqd`: clamp requeue timeout to range instead of dropping connection (thanks @tsholmes) * #891 - `nsqd`: fix race when client subscribes to ephemeral topic or channel while it is being cleaned up (reported by @slayercat) * #927 - `nsqd`: fix deflate level handling * #934 - `nsqd`: fix channel shutdown flush race * #935 - `nsq_to_file`: fix connection leaks when using `--topic-pattern` (thanks @jxskiss) * #951 - mention docker images and binaries for additional platforms in README (thanks @DAXaholic) * #950 - `nsqlookupd`: close connection when magic read fails (thanks @yc90s) * #971 - `nsqd`: fix some races getting ChannelStats (thanks @daroot) * #988 - `nsqd`: fix e2e timings config example, add range validation (thanks @protoss-player) * #991 - `nsq_tail`: logging to stderr (only nsq messages to stdout) * #1000 - `nsq_to_http`: fix http connect/request timeout flags (thanks @kamyanskiy) * #993/#1008 - `nsqd`: fix possible lookupd-identify-error busy-loop (reported by @andyxning) * #1005 - `nsqadmin`: fix typo "Delfate" in connection attrs list (thanks @arussellsaw) * #1032 - `nsqd`: fix loading metadata with messages queued on un-paused topic with multiple channels (thanks @michaelyou) * #1004 - `nsqlookupd`: exit with error when failed to listen on ports (thanks @stephens2424) * #1068 - `nsqadmin`: fix html escaping for large_graph url inside javascript * misc test suite improvements and updates (go versions, tls certs, ...) ### 1.0.0-compat - 2017-03-21 **Upgrading from 0.3.8**: Numerous backwards incompatible changes: * Deprecated `nsqd` features removed: * Pre-V1 HTTP endpoints / response format: * `/{m,}put` (use `/{m,}pub`) * `/{create,delete,empty,pause,unpause}_{topic,channel}` (use `/{topic,channel}/`) * `--max-message-size` flag (use `--max-msg-size`) * `V2` protocol `IDENTIFY` command `short_id` and `long_id` properties (use `client_id`, `hostname`, and `user_agent`) * `/stats` HTTP response `name` property (use `client_id`) * Deprecated `nsqlookupd` features removed: * Pre-V1 HTTP endpoints / response format: * `/{create,delete}_{topic,channel}` (use `/{topic,channel}/`) * `/tombstone_topic_producer` (use `/topic/tombstone`) * Deprecated `nsqadmin` features removed: * `--template-dir` flag (not required, templates are compiled into binary) * `--use-statsd-prefixes` flag (use `--statsd-counter-format` and `--statsd-gauge-format`) * `nsq_stat` `--status-every` flag (use `--interval`) * `--reader-opt` on all binaries that had this flag (use `--consumer-opt`) * `nsq_to_file` `--gzip-compression` flag (use `--gzip-level`) * `nsq_to_http` `--http-timeout` and `--http-timeout-ms` flags (use `--http-connect-timeout` and `--http-request-timeout`) * `nsq_to_http` `--round-robin` flag (use `--mode=round-robin`) * `nsq_to_http` `--max-backoff-duration` flag (use `--consumer-opt=max_backoff_duration,X`) * `nsq_to_http` `--throttle-fraction` flag (use `--sample=X`) * `nsq_to_nsq` `--max-backoff-duration` flag (use `--consumer-opt=max_backoff_duration,X`) * `nsqd` `--worker-id` deprecated in favor of `--node-id` (to be fully removed in subsequent release) This is a compatibility release that drops a wide range of previously deprecated features (#367) while introducing some new deprecations (#844) that we intend to fully remove in a subsequent 1.0 release. Of note, all of the pre-1.0 HTTP endpoints (and response formats) are gone. Any clients or tools that use these endpoints/response formats won't work with this release. These changes have been available since 0.2.29 (released in July of 2014). Clients wishing to forwards-compatibly upgrade can either use the new endpoints or send the following header: Accept: application/vnd.nsq version=1.0 Also, many command line flags have been removed — in almost all cases an alternative is available with a (hopefully) more obvious name. These changes have the same affect on config file option names. On Linux, this release will automatically migrate `nsq..dat` named metadata files to `nsq.dat` in a way that allows users to seamlessly _downgrade_ from this release back to 0.3.8, if necessary. A subsequent release will clean up these convenience symlinks and observe only `nsq.dat`. See the discussion in #741 and the changes #844 for more details. Performance wise, #741 landed which significantly reduces global contention on internal message ID generation, providing a ~1.75x speed improvement on multi-topic benchmarks. Finally, a number of minor issues were resolved spanning contributions from 9 community members! Thanks! Features: * #766 - use `alpine` base image for official Docker container (thanks @kenjones-cisco) * #775 - `nsqadmin:` `/config` API (thanks @kenjones-cisco) * #776 - `nsqadmin`, `nsq_stat`, `nsq_to_file`, `nsq_to_http`: HTTP client connect/request timeouts (thanks @kenjones-cisco) * #777/#778/#783/#785 - improve test coverage (thanks @kenjones-cisco) * #788 - `to_nsq`: add `--rate` message throttling option * #367 - purge deprecated features (see above) * #741 - `nsqd`: per-topic message IDs (multi-topic pub benchmarks up to ~1.75x faster) * #850 - `nsqd`, `nsqlookupd`, `nsqadmin`: add `--log-prefix` option (thanks @ploxiln) * #844 - `nsqd`: deprecate `--worker-id` for `--node-id` and drop ID from `nsqd.dat` file (thanks @ploxiln) Bugs: * #787 - `nsqlookupd`: properly close TCP connection in `IOLoop` (thanks @JackDrogon) * #792 - `nsqdmin`: fix root CA verification (thanks @joshuarubin) * #794 - `nsq_to_file`: require `--topic` or `--topic-pattern` (thanks @judwhite) * #816/#823 - `nsqadmin`: fix handling of IPv6 broadcast addresses (thanks @magnetised) * #805/#832 - `nsqd`: fix requeue and deferred message accounting (thanks @sdbaiguanghe) * #532/#830 - `nsqd`: switch to golang/snappy to fix snappy deadlock * #826/#831/#837/#839 - `nsqd`: fix default `--broadcast-address` and error when `nsqlookupd` reqs fail (thanks @ploxiln @stephensearles) * #822/#835 - `nsqd`: prevent panic in binary `/mpub` (thanks @yangyifeng01) * #841 - `nsqadmin`: allow ctrl/meta+click to open a new tab * #843 - `nsqd`: check for exit before requeing ### 0.3.8 - 2016-05-26 **Upgrading from 0.3.7**: Binaries contain no backwards incompatible changes. This release fixes a critical regression in `0.3.7` that could result in message loss when attempting to cleanly shutdown `nsqd` by sending it a `SIGTERM`. The expected behavior was for it to flush messages in internal buffers to disk before exiting. See #757 and #759 for more details. A few performance improvements landed including #743, which improves channel throughput by ~17%, and #740, which reduces garbage when reading messages from disk. We're now stripping debug info, reducing binary size, in the official binary downloads and Windows binaries are now bundled with the appropriate `.exe` extension (#726 and #751). Features: * #743 - `nsqd`: remove channel `messagePump` * #751 - strip debug info from binaries (thanks @ploxiln) * #740 - `nsqd`: reduce garbage when reading from diskqueue (thanks @dieterbe) Bugs: * #757/#759 - `nsqd`: properly handle `SIGTERM` (thanks @judwhite) * #738 - updates for latest `go-options` * #730 - `nsqd`: diskqueue sync count on both read/write * #734 - `nsqadmin`: make `rate` column work without `--proxy-graphite` (thanks @ploxiln) * #726 - add `.exe` extension to Windows binaries (thanks @ploxiln) * #722 - `nsqadmin`: fix connected duration > `1hr` ### 0.3.7 - 2016-02-23 **Upgrading from 0.3.6**: Binaries contain no backwards incompatible changes. This release has been built with Go 1.6. Highlights include the various work done to reduce `nsqd` lock contention, significantly improving the impact of high load on the `/stats` endpoint, addressing issues with timeouts and failures in `nsqadmin` (#700, #701, #703, #709). Thanks to @judwhite, `nsqd` and `nsqlookupd` now natively support being run as a Windows service (#718). We're also now publishing official Windows releases. `nsqd` will now `flock` its data directory on linux, preventing two `nsqd` from running simultaneously pointed at the same path (#583). On the bugfix side, the most noteworthy change is that `nsqd` will now correctly reset health state on a successful backend write (#671). Features: * #700/#701/#703/#709 - `nsqd`: reduce lock contention (thanks @zachbadgett @absolute8511) * #718 - `nsqd`/`nsqlookupd`: support running as a windows service (thanks @judwhite) * #706 - `nsqd`: support enabling/disabling block profile via HTTP (thanks @absolute8511) * #710 - `nsqd`: support `POST` `/debug/pprof/symbol` (thanks @absolute8511) * #662 - `nsqadmin`: add flags for formatting statsd keys (thanks @kesutton) * #583 - `nsqd`: `flock` `--data-path` on linux * #663 - `nsqd`: optimize GUID generation (thanks @ploxiln) Bugs: * #672 - `nsqd`: fix max size accounting in `diskqueue` (thanks @judwhite) * #671 - `nsqd`: reset health on successful backend write (thanks @judwhite) * #615 - `nsqd`: prevent OOM when reading from `nsqlookupd` peer * #664/#666 - dist.sh/Makefile cleanup (thanks @ploxiln) ### 0.3.6 - 2015-09-24 **Upgrading from 0.3.5**: Binaries contain no backwards incompatible changes. We've adopted the [Contributor Covenant 1.2 Code of Conduct](CODE_OF_CONDUCT.md) (#593). Help us keep NSQ open and inclusive by reading and following this document. We closed a few longstanding issues related to `nsqadmin`, namely (#323, et al.) converting it to an API and single-page app (so that it is _much_ easier to develop), displaying fine-grained errors (#421, #657), and enabling support for `--tls-required` configurations (#396). For `nsqd`, we added support for deferred publishing aka `DPUB` (#293), which allows a producer to specify a duration of time to delay initial delivery of the message. We also addressed performance issues relating to large numbers of topics/channels (#577) by removing some per-channel goroutines in favor of a centralized, periodic, garbage collection approach. In order to provide more flexibility when deploying NSQ in dynamically orchestrated topologies, `nsqd` now supports the ability to configure `nsqlookupd` peers at runtime via HTTP (#601), eliminating the need to restart the daemon. As part of the large `nsqadmin` refactoring, we took the opportunity to cleanup the internals for _all_ of the daemon's HTTP code paths (#601, #610, #612, #641) as well as improving the test suite so that it doesn't leave around temporary files (#553). Features: * #593 - add code of conduct * #323/#631/#632/#642/#421/#649/#650/#651/#652/#654 - `nsqadmin`: convert to API / single-page app * #653 - `nsqadmin`: expand notification context * #293 - `nsqd`: add deferred pub (`DPUB`) * #577 - `nsqd`: drop per-channel queue workers in favor of centralized queue GC * #584 - `nsqlookupd`: improve registration DB performance (thanks @xiaost) * #601 - `nsqd`: HTTP endpoints to dynamically configure `nsqlookupd` peers * #608 - `nsqd`: support for filtering `/stats` to topic/channel (thanks @chrusty) * #601/#610/#612/#641 - improved HTTP internal routing / log HTTP requests * #628 - `nsqd`: clarify help text for `--e2e-processing-latency-percentile` * #640 - switch `--{consumer,producer}-opt` to `nsq.ConfigFlag` Bugs: * #656 - `nsqadmin`: update `statsd` prefix to `stats.counters` * #421/#657 - `nsqadmin`: display upstream/partial errors * #396 - `nsqdamin`/`nsqd`: support for `--tls-required` * #558 - don't overwrite docker root FS * #582 - `nsqd`: ignore benign EOF errors * #587 - `nsqd`: GUID error handling / catch errors if GUID goes backwards (thanks @mpe) * #586 - `nsqd`: fix valid range for `--worker-id` * #550/#602/#617/#618/#619/#620/#622 - `nsqd`: fix benchmarks (thanks @Dieterbe) * #553 - cleanup test dirs * #600 - `nsqd`: enforce diskqueue min/max message size (thanks @twmb) ### 0.3.5 - 2015-04-26 **Upgrading from 0.3.3**: Binaries contain no backwards incompatible changes. This is another quick bug fix release to address the broken `nsqadmin` binary in the distribution (see #578). ### 0.3.4 - 2015-04-26 **WARNING**: please upgrade to `v0.3.5` to address the broken `nsqadmin` binary. **Upgrading from 0.3.3**: Binaries contain no backwards incompatible changes. This is a quick bug fix release to fix the outdated `go-nsq` dependency in `v0.3.3` for the bundled utilities (see 6e8504e). ### 0.3.3 - 2015-04-26 **WARNING**: please upgrade to `v0.3.5` to address the outdated `go-nsq` dependency for the bundled utilities and the broken `nsqadmin` binary. **Upgrading from 0.3.2**: Binaries contain no backwards incompatible changes. This release is primarily a bug fix release after cleaning up and reorganizing the codebase. `nsqadmin` is now importable, which paves the way for completing #323. The bundled utilities received a few feature additions and bug fixes (mostly from bug fixes on the `go-nsq` side). Features: * #569 - `nsqadmin`: re-org into importable package * #562 - `nsq_to_{nsq,http}`: add `epsilon-greedy` mode (thanks @twmb) * #547 - `nsqd`: adds `start_time` to `/stats` (thanks @ShawnSpooner) * #544 - `nsq_to_http`: accept any `200` response as success (thanks @mikedewar) * #548 - `nsq_to_http`: read entire request body (thanks @imgix) * #552/#554/#555/#556/#561 - code cleanup and `/internal` package re-org (thanks @cespare) Bugs: * #573 - `nsqd`: don't persist metadata upon startup (thanks @xiaost) * #560 - `nsqd`: do not print `EOF` error when client closes cleanly (thanks @twmb) * #557 - `nsqd`: fix `--tls-required=tcp-https` with `--tls-client-auth-policy` (thanks @twmb) * #545 - enable shell expansion in official Docker image (thanks @paddyforan) NOTE: the bundled utilities are built against [`go-nsq` `v1.0.4`][go-nsq_104] and include all of those features/fixes. [go-nsq_104]: https://github.com/nsqio/go-nsq/releases/tag/v1.0.4 ### 0.3.2 - 2015-02-08 **Upgrading from 0.3.1**: Binaries contain no backwards incompatible changes however as of this release we've updated our official Docker images. We now provide a single Docker image [`nsqio/nsq`](https://registry.hub.docker.com/r/nsqio/nsq/) that includes *all* of the NSQ binaries. We did this for several reasons, primarily because the tagged versions in the previous incarnation were broken (and did not actually pin to a version!). The new image is an order of magnitude smaller, weighing in around 70mb. In addition, the impetus for this quick release is to address a slew of reconnect related bug fixes in the utility apps (`nsq_to_nsq`, `nsq_to_file`, etc.), for details see the [`go-nsq` `v1.0.3` release notes](https://github.com/nsqio/go-nsq/releases/tag/v1.0.3). Features: * #534/#539/#540 - single Dockerfile approach (thanks @paddyforan) Bugs: * #529 - nsqadmin: fix more `#ephemeral` topic deletion issues * #530 - nsqd: fix the provided sample config file (thanks @jnewmano) * #538 - nsqd: fix orphaned ephemeral channels (thanks @adamsathailo) ### 0.3.1 - 2015-01-21 **Upgrading from 0.3.0**: No backwards incompatible changes. This release contains minor bug fixes and feature additions. There are a number of functionality improvements to the `nsq_stat` and `nsq_to_file` helper applications (and general support for `#ephemeral` topics, broken in `0.2.30`). Additionally, the TLS options continue to improve with support for setting `--tls-min-version` and a work-around for a bug relating to `TLS_FALLBACK_SCSV` ([to be fixed in Go 1.5](https://go-review.googlesource.com/#/c/1776/)). Features: * #527 - nsq_stat: deprecate `--status-every` in favor of `--interval` * #524 - nsq_stat: add `--count` option (thanks @nordicdyno) * #518 - nsqd: set defaults for `--tls-min-version` and set TLS max version to 1.2 * #475/#513/#518 - nsqd: `--tls-required` can be disabled for HTTP / add `--tls-min-version` (thanks @twmb) * #496 - nsq_to_file: add `` to filename and rotation by size/interval (thanks @xiaost) * #507 - nsq_stat: add rates (thanks @xiaost) * #505 - nsqd: speed up failure path of `BytesToBase10` (thanks @iand) Bugs: * #522 - nsqadmin: fix `#ephemeral` topic deletion issues * #509 - nsqd: fix `diskqueue` atomic rename on Windows (thanks @allgeek) * #479 - nsqd: return `output_buffer_*` resolved settings in `IDENTIFY` response (thanks @tj) ### 0.3.0 - 2014-11-18 **Upgrading from 0.2.31**: No backwards incompatible changes. This release includes a slew of bug fixes and few key feature additions. The biggest functional change is that `nsqd` no longer decrements its `RDY` count for clients. This means that client libraries no longer have to periodically re-send `RDY`. For some context, `nsqd` already provided back-pressure due to the fact that a client must respond to messages before receiving new ones. The decremented `RDY` count only made the implementation of the server and client more complex without additional benefit. Now the `RDY` command can be treated as an "on/off" switch. For details see #404 and the associated changes in nsqio/go-nsq#83 and nsqio/pynsq#98. The second biggest change (and oft-requested feature!) is `#ephemeral` topics. Their behavior mirrors that of channels. This feature is incredibly useful for situations where you're using topics to "route" messages to consumers (like RPC) or when a backlog of messages is undesirable. There are now scripts in the `bench` directory that automate the process of running a distributed benchmark. This is a work-in-progress, but it already provides a closer-to-production setup and therefore more accurate results. There's much work to do here! A whole bunch of bugs were fixed - notably all were 3rd-party contributions! Thanks! * #305 - `#ephemeral` topics * #404/#459 - don't decr `RDY` / send `RDY` before `FIN`/`REQ` * #472 - improve `nsqd` `diskqueue` sync strategies * #488 - ability to filter topics by regex in `nsq_to_file` (thanks @lxfontes) * #438 - distributed pub-sub benchmark scripts * #448 - better `nsqd` `IOLoop` logging (thanks @rexposadas) * #458 - switch to [gpm](https://github.com/pote/gpm) for builds Bugs: * #493 - ensure all `nsqd` `Notify()` goroutines have exited prior to shutdown (thanks @allgeek) * #492 - ensure `diskqueue` syncs at end of benchmarks (thanks @Dieterbe) * #490 - de-flake `TestPauseMetadata` (thanks @allgeek) * #486 - require ports to be specified for daemons (thanks @jnewmano) * #482 - better bash in `dist.sh` (thanks @losinggeneration) * #480 - fix panic when `nsqadmin` checks stats for missing topic (thanks @jnewmano) * #469 - fix panic when misbehaving client sends corrupt command (thanks @prio) * #461 - fix panic when `nsqd` decodes corrupt message data (thanks @twmb) * #454/#455 - fix 32-bit atomic ops in `nsq_to_nsq`/`nsq_to_http` (thanks @leshik) * #451 - fix `go get` compatibility (thanks @adams-sarah) ### 0.2.31 - 2014-08-26 **Upgrading from 0.2.30**: No backwards incompatible changes. This release includes a few key changes. First, we improved feedback and back-pressure when `nsqd` writes to disk. Previously this was asynchronous and would result in clients not knowing that their `PUB` had failed. Interestingly, this refactoring improved performance of `PUB` by 41%, by removing the topic's goroutine responsible for message routing in favor of `N:N` Go channel communication. For details see #437. @paddyforan contributed official Dockerfiles that are now built automatically via Docker Hub. Please begin to use (and improve these) as the various older images we had been maintaining will be deprecated. The utility apps deprecated the `--reader-opt` flag in favor of `--consumer-opt` and `nsq_to_nsq` and `to_nsq` received a `--producer-opt` flag, for configuring details of the connection publishing to `nsqd`. Additionally, it is now possible to configure client side TLS certificates via `tls_cert` and `tls_key` opts. As usual, we fixed a few minor bugs, see below for details. New Features / Enhancements: * #422/#437 - `nsqd`: diskqueue error feedback/backpressure (thanks @boyand) * #412 - official Dockerfiles for `nsqd`, `nsqlookupd`, `nsqadmin` (thanks @paddyforan) * #442 - utilities: add `--consumer-opt` alias for `--reader-opt` and add `--producer-opt` to `nsq_to_nsq` (also support configuration of `tls_cert` and `tls_key`) * #448 - `nsqd`: improve IOLoop error messages (thanks @rexposadas) Bugs: * #440 - `nsqd`: fixed statsd GC stats reporting (thanks @jphines) * #434/#435 - refactored/stabilized tests and logging * #429 - `nsqd`: improve handling/documentation of `--worker-id` (thanks @bschwartz) * #428 - `nsqd`: `IDENTIFY` should respond with materialized `msg_timeout` (thanks @visionmedia) ### 0.2.30 - 2014-07-28 **Upgrading from 0.2.29**: No backwards incompatible changes. **IMPORTANT**: this is a quick bug-fix release to address a panic in `nsq_to_nsq` and `nsq_to_http`, see #425. New Features / Enhancements: * #417 - `nsqadmin`/`nsqd`: expose TLS connection state * #425 - `nsq_to_nsq`/`nsq_to_file`: display per-destination-address timings Bugs: * #425 - `nsq_to_nsq`/`nsq_to_file`: fix shared mutable state panic ### 0.2.29 - 2014-07-25 **Upgrading from 0.2.28**: No backwards incompatible changes. This release includes a slew of new features and bug fixes, with contributions from 8 members of the community, thanks! The most important new feature is authentication (the `AUTH` command for `nsqd`), added in #356. When `nsqd` is configured with an `--auth-http-address` it will require clients to send the `AUTH` command. The `AUTH` command body is opaque to `nsqd`, it simply passes it along to the configured auth daemon which responds with well formed JSON, indicating which topics/channels and properties on those entities are accessible to that client (rejecting the client if it accesses anything prohibited). For more details, see [the spec](https://nsq.io/clients/tcp_protocol_spec.html) or [the `nsqd` guide](https://nsq.io/components/nsqd.html#auth). Additionally, we've improved performance in a few areas. First, we refactored in-flight handling in `nsqd` to reduce garbage creation and improve baseline performance 6%. End-to-end processing latency calculations are also significantly faster, thanks to improvements in the [`perks`](https://github.com/bmizerany/perks/pulls/7) package. HTTP response formats have been improved (removing the redundant response wrapper) and cleaning up some of the endpoint namespaces. This change is backwards compatible. Clients wishing to move towards the new response format can either use the new endpoint names or send the following header: Accept: application/vnd.nsq version=1.0 Other changes including officially bumping the character limit for topic and channel names to 64 (thanks @svmehta), making the `REQ` timeout limit configurable in `nsqd` (thanks @AlphaB), and compiling static asset dependencies into `nsqadmin` to simplify deployment (thanks @crossjam). Finally, `to_nsq` was added to the suite of bundled apps. It takes a stdin stream and publishes to `nsqd`, an extremely flexible solution (thanks @matryer)! As for bugs, they're mostly minor, see the pull requests referenced in the section below for details. New Features / Enhancements: * #304 - apps: added `to_nsq` for piping stdin to NSQ (thanks @matryer) * #406 - `nsqadmin`: embed external static asset dependencies (thanks @crossjam) * #389 - apps: report app name and version via `user_agent` * #378/#390 - `nsqd`: improve in-flight message handling (6% faster, GC reduction) * #356/#370/#386 - `nsqd`: introduce `AUTH` * #358 - increase topic/channel name max length to 64 (thanks @svmehta) * #357 - remove internal `go-nsq` dependencies (GC reduction) * #330/#366 - version HTTP endpoints, simplify response format * #352 - `nsqd`: make `REQ` timeout limit configurable (thanks @AlphaB) * #340 - `nsqd`: bump perks dependency (E2E performance improvement, see 25086e4) Bugs: * #384 - `nsqd`: fix statsd GC time reporting * #407 - `nsqd`: fix double `TOUCH` and use of client's configured msg timeout * #392 - `nsqadmin`: fix HTTPS warning (thanks @juliangruber) * #383 - `nsqlookupd`: fix race on last update timestamp * #385 - `nsqd`: properly handle empty `FIN` * #365 - `nsqd`: fix `IDENTIFY` `msg_timeout` response (thanks @visionmedia) * #345 - `nsq_to_file`: set proper permissions on new directories (thanks @bschwartz) * #338 - `nsqd`: fix windows diskqueue filenames (thanks @politician) ### 0.2.28 - 2014-04-28 **Upgrading from 0.2.27**: No backwards incompatible changes. We've deprecated the `short_id` and `long_id` options in the `IDENTIFY` command in favor of `client_id` and `hostname`, which more accurately reflect the data typically used. This release includes a few important new features, in particular enhanced `nsqd` TLS support thanks to a big contribution by @chrisroberts. You can now *require* that clients negotiate TLS with `--tls-required` and you can configure a client certificate policy via `--tls-client-auth-policy` (`require` or `require-verify`): * `require` - the client must offer a certificate, otherwise rejected * `require-verify` - the client must offer a valid certificate according to the default CA or the chain specified by `--tls-root-ca-file`, otherwise rejected This can be used as a form of client authentication. Additionally, `nsqd` is now structured such that it is importable in other Go applications via `github.com/nsqio/nsq/nsqd`, thanks to @kzvezdarov. Finally, thanks to @paddyforan, `nsq_to_file` can now archive *multiple* topics or optionally archive *all* discovered topics (by specifying no `--topic` params and using `--lookupd-http-address`). New Features / Enhancements: * #334 - `nsq_to_file` can archive many topics (thanks @paddyforan) * #327 - add `nsqd` TLS client certificate verification policy, ability to require TLS, and HTTPS support (thanks @chrisroberts) * #325 - make `nsqd` importable (`github.com/nsqio/nsq/nsqd`) (thanks @kzvezdarov) * #321 - improve `IDENTIFY` options (replace `short_id` and `long_id` with `client_id` and `hostname`) * #319 - allow path separator in `nsq_to_file` filenames (thanks @jsocol) * #324 - display memory depth and total depth in `nsq_stat` Bug Fixes: * nsqio/go-nsq#19 and nsqio/go-nsq#29 - fix deadlocks on `nsq.Reader` connection close/exit, this impacts the utilities packaged with the NSQ binary distribution such as `nsq_to_file`, `nsq_to_http`, `nsq_to_nsq` and `nsq_tail`. * #329 - use heartbeat interval for write deadline * #321/#326 - improve benchmarking tests * #315/#318 - fix test data races / flakiness ### 0.2.27 - 2014-02-17 **Upgrading from 0.2.26**: No backwards incompatible changes. We deprecated `--max-message-size` in favor of `--max-msg-size` for consistency with the rest of the flag names. IMPORTANT: this is another quick bug-fix release to address an issue in `nsqadmin` where templates were incompatible with older versions of Go (pre-1.2). * #306 - fix `nsqadmin` template compatibility (and formatting) * #310 - fix `nsqadmin` behavior when E2E stats are disabled * #309 - fix `nsqadmin` `INVALID_ERROR` on node page tombstone link * #311/#312 - fix `nsqd` client metadata race condition and test flakiness * #314 - fix `nsqd` test races (run w/ `-race` and `GOMAXPROCS=4`) deprecate `--max-message-size` ### 0.2.26 - 2014-02-06 **Upgrading from 0.2.25**: No backwards incompatible changes. IMPORTANT: this is a quick bug-fix release to address a regression identified in `0.2.25` where `statsd` prefixes were broken when using the default (or any) prefix that contained a `%s` for automatic host replacement. * #303 - fix `nsqd` `--statsd-prefix` when using `%s` host replacement ### 0.2.25 - 2014-02-05 **Upgrading from 0.2.24**: No backwards incompatible changes. This release adds several commonly requested features. First, thanks to [@elubow](https://twitter.com/elubow) you can now configure your clients to sample the stream they're subscribed to. To read more about the details of the implementation see #286 and the original discussion in #223. Eric also contributed an improvement to `nsq_tail` to add the ability to tail the last `N` messages and exit. We added config file support ([TOML](https://github.com/mojombo/toml/blob/master/README.md)) for `nsqd`, `nsqlookupd`, and `nsqadmin` - providing even more deployment flexibility. Example configs are in the `contrib` directory. Command line arguments override the equivalent option in the config file. We added the ability to pause a *topic* (it is already possible to pause individual *channels*). This functionality stops all message flow from topic to channel for *all channels* of a topic, queueing at the topic level. This enables all kinds of interesting possibilities like atomic channel renames and trivial infrastructure wide operations. Finally, we now compile the static assets used by `nsqadmin` into the binary, simplifying deployment. This means that `--template-dir` is now deprecated and will be removed in a future release and you can remove the templates you previously deployed and maintained. New Features / Enhancements: * #286 - add client `IDENTIFY` option to sample a % of messages * #279 - add TOML config file support to `nsqd`, `nsqlookupd`, and `nsqadmin` * #263 - add ability to pause a topic * #291 - compile templates into `nsqadmin` binary * #285/#288 - `nsq_tail` support for `-n #` to get recent # messages * #287/#294 - display client `IDENTIFY` attributes in `nsqadmin` (sample rate, TLS, compression) * #189/#296 - add client user agent to `nsqadmin`` * #297 - add `nsq_to_nsq` JSON message filtering options ### 0.2.24 - 2013-12-07 **Upgrading from 0.2.23**: No backwards incompatible changes. However, as you'll see below, quite a few command line flags to the utility apps (`nsq_to_http`, `nsq_to_file`, `nsq_to_http`) were deprecated and will be removed in the next release. Please use this release to transition over to the new ones. NOTE: we are now publishing additional binaries built against go1.2 The most prominent addition is the tracking of end-to-end message processing percentiles. This measures the amount of time it's taking from `PUB` to `FIN` per topic/channel. The percentiles are configurable and, because there is *some* overhead in collecting this data, it can be turned off entirely. Please see [the section in the docs](https://nsq.io/components/nsqd.html) for implementation details. Additionally, the utility apps received comprehensive support for all configurable reader options (including compression, which was previously missing). This necessitated a bit of command line flag cleanup, as follows: #### nsq_to_file * deprecated `--gzip-compression` in favor of `--gzip-level` * deprecated `--verbose` in favor of `--reader-opt=verbose` #### nsq_to_http * deprecated `--throttle-fraction` in favor of `--sample` * deprecated `--http-timeout-ms` in favor of `--http-timeout` (which is a *duration* flag) * deprecated `--verbose` in favor of `--reader-opt=verbose` * deprecated `--max-backoff-duration` in favor of `--reader-opt=max_backoff_duration=X` #### nsq_to_nsq * deprecated `--verbose` in favor of `--reader-opt=verbose` * deprecated `--max-backoff-duration` in favor of `--reader-opt=max_backoff_duration=X` New Features / Enhancements: * #280 - add end-to-end message processing latency metrics * #267 - comprehensive reader command line flags for utilities ### 0.2.23 - 2013-10-21 **Upgrading from 0.2.22**: No backwards incompatible changes. We now use [godep](https://github.com/kr/godep) in order to achieve reproducible builds with pinned dependencies. If you're on go1.1+ you can now just use `godep get github.com/nsqio/nsq/...`. This release includes `nsqd` protocol compression feature negotiation. [Snappy](https://code.google.com/p/snappy/) and [Deflate](http://en.wikipedia.org/wiki/DEFLATE) are supported, clients can choose their preferred format. `--statsd-prefix` can now be used to modify the prefix for the `statsd` keys generated by `nsqd`. This is useful if you want to add datacenter prefixes or remove the default host prefix. Finally, this release includes a "bug" fix that reduces CPU usage for `nsqd` with many clients by choosing a more reasonable default for a timer used in client output buffering. For more details see #236. New Features / Enhancements: * #266 - use godep for reproducible builds * #229 - compression (Snappy/Deflate) feature negotiation * #241 - binary support for HTTP /mput * #269 - add --statsd-prefix flag Bug Fixes: * #278 - fix nsqd race for client subscription cleanup (thanks @simplereach) * #277 - fix nsqadmin counter page * #275 - stop accessing simplejson internals * #274 - nsqd channel pause state lost during unclean restart (thanks @hailocab) * #236 - reduce "idle" CPU usage by 90% with large # of clients ### 0.2.22 - 2013-08-26 **Upgrading from 0.2.21**: message timestamps are now officially nanoseconds. The protocol docs always stated this however `nsqd` was actually sending seconds. This may cause some compatibility issues for client libraries/clients that were taking advantage of this field. This release also introduces support for TLS feature negotiation in `nsqd`. Clients can optionally enable TLS by using the appropriate handshake via the `IDENTIFY` command. See #227. Significant improvements were made to the HTTP publish endpoints and in flight message handling to reduce GC pressure and eliminate memory abuse vectors. See #242, #239, and #245. This release also includes a new utility `nsq_to_nsq` for performant, low-latency, copying of an NSQ topic over the TCP protocol. Finally, a whole suite of debug HTTP endpoints were added (and consolidated) under the `/debug/pprof` namespace. See #238, #248, and #252. As a result `nsqd` now supports *direct* profiling via Go's `pprof` tool, ie: $ go tool pprof --web http://ip.address:4151/debug/pprof/heap New Features / Enhancements: * #227 - TLS feature negotiation * #238/#248/#252 - support for more HTTP debug endpoints * #256 - `nsqadmin` single node view (with GC/mem graphs) * #255 - `nsq_to_nsq` utility for copying a topic over TCP * #230 - `nsq_to_http` takes `--content-type` flag (thanks @michaelhood) * #228 - `nsqadmin` displays tombstoned topics in the `/nodes` list * #242/#239/#245 - reduced GC pressure for inflight and `/mput` Bug Fixes: * #260 - `tombstone_topic_producer` action in `nsqadmin` missing node info * #244 - fix 64bit atomic alignment issues on 32bit platforms * #251 - respect configured limits for HTTP publishing * #247 - publish methods should not allow 0 length messages * #231/#259 - persist `nsqd` metadata on topic/channel changes * #237 - fix potential memory leaks with retained channel references * #232 - message timestamps are now nano * #228 - `nsqlookupd`/`nsqadmin` would display inactive nodes in `/nodes` list * #216 - fix edge cases in `nsq_to_file` that caused empty files ### 0.2.21 - 2013-06-07 **Upgrading from 0.2.20**: there are no backward incompatible changes in this release. This release introduces a significant new client feature as well as a slew of consistency and recovery improvements to diskqueue. First, we expanded the feature negotiation options for clients. There are many cases where you want different output buffering semantics from `nsqd` to your client. You can now control both output buffer size and the output buffer timeout via new fields in the `IDENTIFY` command. You can even disable output buffering if low latency is a priority. You can now specify a duration between fsyncs via `--sync-timeout`. This is a far better way to manage when the process fsyncs messages to disk (vs the existing `--sync-every` which is based on # of messages). `--sync-every` is now considered a deprecated option and will be removed in a future release. Finally, `0.2.20` introduced a significant regression in #176 where a topic would not write messages to its channels. It is recommended that all users running `0.2.20` upgrade to this release. For additional information see #217. New Features / Enhancements: * #214 - add --sync-timeout for time based fsync, improve when diskqueue syncs * #196 - client configurable output buffering * #190 - nsq_tail generates a random #ephemeral channel Bug Fixes: * #218/#220 - expose --statsd-interval for nsqadmin to handle non 60s statsd intervals * #217 - fix new topic channel creation regression from #176 (thanks @elubow) * #212 - dont use port in nsqadmin cookies * #214 - dont open diskqueue writeFile with O_APPEND * #203/#211 - diskqueue depth accounting consistency * #207 - failure to write a heartbeat is fatal / reduce error log noise * #206 - use broadcast address for statsd prefix * #205 - cleanup example utils exit ### 0.2.20 - 2013-05-13 **Upgrading from 0.2.19**: there are no backward incompatible changes in this release. This release adds a couple of convenient features (such as adding the ability to empty a *topic*) and continues our work to reduce garbage produced at runtime to relieve GC pressure in the Go runtime. `nsqd` now has two new flags to control the max value clients can use to set their heartbeat interval as well as adjust a clients maximum RDY count. This is all set/communicated via `IDENTIFY`. `nsqadmin` now displays `nsqd` -> `nsqlookupd` connections in the "nodes" view. This is useful for visualizing how the topology is connected as well as situations where `--broadcast-address` is being used incorrectly. `nsq_to_http` now has a "host pool" mode where upstream state will be adjusted based on successful/failed requests and for failures, upstreams will be exponentially backed off. This is an incredibly useful routing mode. As for bugs, we fixed an issue where "fatal" client errors were not actually being treated as fatal. Under certain conditions deleting a topic would not clean up all of its files on disk. There was a reported issue where the `--data-path` was not writable by the process and this was only discovered after message flow began. We added a writability check at startup to improve feedback. Finally. `deferred_count` was being sent as a counter value to statsd, it should be a gauge. New Features / Enhancements: * #197 - nsqadmin nodes list improvements (show nsqd -> lookupd conns) * #192 - add golang runtime version to daemon version output * #183 - ability to empty a topic * #176 - optimizations to reduce garbage, copying, locking * #184 - add table headers to nsqadmin channel view (thanks @elubow) * #174/#186 - nsq_to_http hostpool mode and backoff control * #173/#187 - nsq_stat utility for command line introspection * #175 - add nsqd --max-rdy-count configuration option * #178 - add nsqd --max-heartbeat-interval configuration option Bug Fixes: * #198 - fix fatal errors not actually being fatal * #195 - fix delete topic does not delete all diskqueue files * #193 - fix data race in channel requeue * #185 - ensure that --data-path is writable on startup * #182 - fix topic deletion ordering to prevent race conditions with lookupd/diskqueue * #179 - deferred_count as gauge for statsd * #173/#188/#191 - fix nsqadmin counter template error; fix nsqadmin displaying negative rates ### 0.2.19 - 2013-04-11 **Upgrading from 0.2.18**: there are no backward incompatible changes in this release. This release is a small release that introduces one major client side feature and resolves one critical bug. `nsqd` clients can now configure their own heartbeat interval. This is important because as of `0.2.18` *all* clients (including producers) received heartbeats by default. In certain cases receiving a heartbeat complicated "simple" clients that just wanted to produce messages and not handle asynchronous responses. This gives flexibility for the client to decide how it would like behave. A critical bug was discovered where emptying a channel would leave client in-flight state inconsistent (it would not zero) which limited deliverability of messages to those clients. New Features / Enhancements: * #167 - 'go get' compatibility * #158 - allow nsqd clients to configure (or disable) heartbeats Bug Fixes: * #171 - fix race conditions identified testing against go 1.1 (scheduler improvements) * #160 - empty channel left in-flight count inconsistent (thanks @dmarkham) ### 0.2.18 - 2013-02-28 **Upgrading from 0.2.17**: all V2 clients of nsqd now receive heartbeats (previously only clients that subscribed would receive heartbeats, excluding TCP *producers*). **Upgrading from 0.2.16**: follow the notes in the 0.2.17 changelog for upgrading from 0.2.16. Beyond the important note above regarding heartbeats this release includes `nsq_tail`, an extremely useful utility application that can be used to introspect a topic on the command line. If statsd is enabled (and graphite in `nsqadmin`) we added the ability to retrieve rates for display in `nsqadmin`. We resolved a few critical issues with data consistency in `nsqlookupd` when channels and topics are deleted. First, deleting a topic would cause that producer to disappear from `nsqlookupd` for all topics. Second, deleting a channel would cause that producer to disappear from the topic list in `nsqlookupd`. New Features / Enhancements: * #131 - all V2 nsqd clients get heartbeats * #154 - nsq_tail example reader * #143 - display message rates in nsqadmin Bug Fixes: * #148 - store tombstone data per registration in nsqlookupd * #153 - fix large graph formulas in nsqadmin * #150/#151 - fix topics disappearing from nsqlookupd when channels are deleted ### 0.2.17 - 2013-02-07 **Upgrading from 0.2.16**: IDENTIFY and SUB now return success responses (they previously only responded to errors). The official Go and Python libraries are forwards/backwards compatible with this change however 3rd party client libraries may not be. **Upgrading from 0.2.15**: in #132 deprecations in SUB were removed as well as support for the old, line oriented, `nsqd` metadata file format. For these reasons you should upgrade to `0.2.16` first. New Features / Enhancements: * #119 - add TOUCH command to nsqd * #142 - add --broadcast-address flag to nsqd/nsqadmin (thanks @dustismo) * #135 - atomic MPUB * #133 - improved protocol fatal error handling and responses; IDENTIFY/SUB success responses * #118 - switch nsqadmin actions to POST and require confirmation * #117/#147 - nsqadmin action POST notifications * #122 - configurable msg size limits * #132 - deprecate identify in SUB and old nsqd metadata file format Bug Fixes: * #144 - empty channel should clear inflight/deferred messages * #140 - fix MPUB protocol documentation * #139 - fix nsqadmin handling of legacy statsd prefixes for graphs * #138/#145 - fix nsqadmin action redirect handling * #134 - nsqd to nsqlookupd registration fixes * #129 - nsq_to_file gzip file versioning * #106 - nsqlookupd topic producer tombstones * #100 - sane handling of diskqueue read errors * #123/#125 - fix notify related exit deadlock ### 0.2.16 - 2013-01-07 **Upgrading from 0.2.15**: there are no backward incompatible changes in this release. However, this release introduces the `IDENTIFY` command (which supersedes sending metadata along with `SUB`) for clients of `nsqd`. The old functionality will be removed in a future release. * #114 persist paused channels through restart * #121 fix typo preventing compile of bench_reader (thanks @datastream) * #120 fix nsqd crash when empty command is sent (thanks @michaelhood) * #115 nsq_to_file --filename-format --datetime-format parameter and fix * #101 fix topic/channel delete operations ordering * #98 nsqadmin fixes when not using lookupd * #90/#108 performance optimizations / IDENTIFY protocol support in nsqd. For a single consumer of small messages (< 4k) increases throughput ~400% and reduces # of allocations ~30%. * #105 strftime compatible datetime format * #103 nsq_to_http handler logging * #102 compatibility with Go tip * #99 nsq_to_file --gzip flag * #95 proxy graphite requests through nsqadmin * #93 fix nqd API response for no topics * #92 graph rendering options * #86 nsq_to_http Content-Length headers * #89 gopkg doc updates * #88 move pynsq to it's own repo * #81/#87 reader improvements / introduced MPUB. Fix bug for mem-queue-size < 10 * #76 statsd/graphite support * #75 administrative ability to create topics and channels ### 0.2.15 - 2012-10-25 * #84 fix lookupd hanging on to ephemeral channels w/ no producers * #82 add /counter page to nsqadmin * #80 message size benchmark * #78 send Content-Length for nsq_to_http requests * #57/#83 documentation updates ### 0.2.14 - 2012-10-19 * #77 ability to pause a channel (includes bugfix for message pump/diskqueue) * #74 propagate all topic changes to lookupd * #65 create binary releases ### 0.2.13 - 2012-10-15 * #70 deadlined nsq_to_http outbound requests * #69/#72 improved nsq_to_file sync strategy * #58 diskqueue metadata bug and refactoring ### 0.2.12 - 2012-10-10 * #63 consolidated protocol V1 into V2 and fixed PUB bug * #61 added a makefile for simpler building * #55 allow topic/channel names with `.` * combined versions for all binaries ### 0.2.7 - 0.2.11 * Initial public release. ## go-nsq Client Library * #264 moved **go-nsq** to its own [repository](https://github.com/nsqio/go-nsq) ## pynsq Python Client Library * #88 moved **pynsq** to its own [repository](https://github.com/nsqio/pynsq) ================================================ FILE: Dockerfile ================================================ FROM golang:alpine AS build RUN apk update && apk add make gcc musl-dev RUN mkdir -p /go/src/github.com/nsqio/nsq COPY . /go/src/github.com/nsqio/nsq WORKDIR /go/src/github.com/nsqio/nsq RUN CGO_ENABLED=0 make BLDDIR=/tmp/nsq PREFIX=/opt/nsq BLDFLAGS='-ldflags="-s -w"' install FROM alpine:latest EXPOSE 4150 4151 4160 4161 4170 4171 RUN mkdir -p /data WORKDIR /data # Optional volumes (explicitly configure with "docker run -v ...") # /data - used by nsqd for persistent storage across restarts # /etc/ssl/certs - for SSL Root CA certificates from host COPY --from=build /opt/nsq/bin/ /usr/local/bin/ RUN ln -s /usr/local/bin/*nsq* / \ && ln -s /usr/local/bin/*nsq* /bin/ ================================================ FILE: LICENSE ================================================ Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. ================================================ FILE: Makefile ================================================ PREFIX=/usr/local BINDIR=${PREFIX}/bin DESTDIR= BLDDIR=build BLDFLAGS= EXT= ifeq (${GOOS},windows) EXT=.exe endif APPS = nsqd nsqlookupd nsqadmin nsq_to_nsq nsq_to_file nsq_to_http nsq_tail nsq_stat to_nsq all: $(APPS) $(BLDDIR)/nsqd: $(wildcard apps/nsqd/*.go nsqd/*.go nsq/*.go internal/*/*.go) $(BLDDIR)/nsqlookupd: $(wildcard apps/nsqlookupd/*.go nsqlookupd/*.go nsq/*.go internal/*/*.go) $(BLDDIR)/nsqadmin: $(wildcard apps/nsqadmin/*.go nsqadmin/*.go nsqadmin/templates/*.go internal/*/*.go) $(BLDDIR)/nsq_to_nsq: $(wildcard apps/nsq_to_nsq/*.go nsq/*.go internal/*/*.go) $(BLDDIR)/nsq_to_file: $(wildcard apps/nsq_to_file/*.go nsq/*.go internal/*/*.go) $(BLDDIR)/nsq_to_http: $(wildcard apps/nsq_to_http/*.go nsq/*.go internal/*/*.go) $(BLDDIR)/nsq_tail: $(wildcard apps/nsq_tail/*.go nsq/*.go internal/*/*.go) $(BLDDIR)/nsq_stat: $(wildcard apps/nsq_stat/*.go internal/*/*.go) $(BLDDIR)/to_nsq: $(wildcard apps/to_nsq/*.go internal/*/*.go) $(BLDDIR)/%: @mkdir -p $(dir $@) go build ${BLDFLAGS} -o $@ ./apps/$* $(APPS): %: $(BLDDIR)/% clean: rm -fr $(BLDDIR) .PHONY: install clean all .PHONY: $(APPS) install: $(APPS) install -m 755 -d ${DESTDIR}${BINDIR} for APP in $^ ; do install -m 755 ${BLDDIR}/$$APP ${DESTDIR}${BINDIR}/$$APP${EXT} ; done ================================================ FILE: README.md ================================================

  • Source: https://github.com/nsqio/nsq
  • Issues: https://github.com/nsqio/nsq/issues
  • Mailing List: nsq-users@googlegroups.com
  • IRC: #nsq on freenode
  • Docs: https://nsq.io
  • Twitter: @nsqio

[![Build Status](https://github.com/nsqio/nsq/workflows/tests/badge.svg)](https://github.com/nsqio/nsq/actions) [![GitHub release](https://img.shields.io/github/release/nsqio/nsq.svg)](https://github.com/nsqio/nsq/releases/latest) [![Coverage Status](https://coveralls.io/repos/github/nsqio/nsq/badge.svg?branch=master)](https://coveralls.io/github/nsqio/nsq?branch=master) **NSQ** is a realtime distributed messaging platform designed to operate at scale, handling billions of messages per day. It promotes *distributed* and *decentralized* topologies without single points of failure, enabling fault tolerance and high availability coupled with a reliable message delivery guarantee. See [features & guarantees][features_guarantees]. Operationally, **NSQ** is easy to configure and deploy (all parameters are specified on the command line and compiled binaries have no runtime dependencies). For maximum flexibility, it is agnostic to data format (messages can be JSON, MsgPack, Protocol Buffers, or anything else). Official Go and Python libraries are available out of the box (as well as many other [client libraries][client_libraries]), and if you're interested in building your own, there's a [protocol spec][protocol]. We publish [binary releases][installing] for Linux, Darwin, FreeBSD and Windows, as well as an official [Docker image][docker_deployment]. NOTE: master is our *development* branch and may not be stable at all times. ## In Production               
              
              
              
              
        

## Code of Conduct Help us keep NSQ open and inclusive. Please read and follow our [Code of Conduct](CODE_OF_CONDUCT.md). ## Authors NSQ was designed and developed by Matt Reiferson ([@imsnakes][snakes_twitter]) and Jehiah Czebotar ([@jehiah][jehiah_twitter]) but wouldn't have been possible without the support of [Bitly][bitly], maintainers ([Pierce Lopez][pierce_github]), and all our [contributors][contributors]. Logo created by Wolasi Konu ([@kisalow][wolasi_twitter]). [protocol]: https://nsq.io/clients/tcp_protocol_spec.html [installing]: https://nsq.io/deployment/installing.html [docker_deployment]: https://nsq.io/deployment/docker.html [snakes_twitter]: https://twitter.com/imsnakes [jehiah_twitter]: https://twitter.com/jehiah [bitly]: https://bitly.com [features_guarantees]: https://nsq.io/overview/features_and_guarantees.html [contributors]: https://github.com/nsqio/nsq/graphs/contributors [client_libraries]: https://nsq.io/clients/client_libraries.html [wolasi_twitter]: https://twitter.com/kisalow [pierce_github]: https://github.com/ploxiln ================================================ FILE: apps/nsq_stat/nsq_stat.go ================================================ // This is a utility application that polls /stats for all the producers // of the specified topic/channel and displays aggregate stats package main import ( "errors" "flag" "fmt" "log" "os" "os/signal" "strconv" "strings" "syscall" "time" "github.com/nsqio/nsq/internal/app" "github.com/nsqio/nsq/internal/clusterinfo" "github.com/nsqio/nsq/internal/http_api" "github.com/nsqio/nsq/internal/version" ) var ( showVersion = flag.Bool("version", false, "print version") topic = flag.String("topic", "", "NSQ topic") channel = flag.String("channel", "", "NSQ channel") interval = flag.Duration("interval", 2*time.Second, "duration of time between polling/printing output") httpConnectTimeout = flag.Duration("http-client-connect-timeout", 2*time.Second, "timeout for HTTP connect") httpRequestTimeout = flag.Duration("http-client-request-timeout", 5*time.Second, "timeout for HTTP request") countNum = numValue{} nsqdHTTPAddrs = app.StringArray{} lookupdHTTPAddrs = app.StringArray{} ) type numValue struct { isSet bool value int } func (nv *numValue) String() string { return "N" } func (nv *numValue) Set(s string) error { value, err := strconv.ParseInt(s, 10, 32) if err != nil { return err } nv.value = int(value) nv.isSet = true return nil } func init() { flag.Var(&nsqdHTTPAddrs, "nsqd-http-address", "nsqd HTTP address (may be given multiple times)") flag.Var(&lookupdHTTPAddrs, "lookupd-http-address", "lookupd HTTP address (may be given multiple times)") flag.Var(&countNum, "count", "number of reports") } func statLoop(interval time.Duration, connectTimeout time.Duration, requestTimeout time.Duration, topic string, channel string, nsqdTCPAddrs []string, lookupdHTTPAddrs []string) { ci := clusterinfo.New(nil, http_api.NewClient(nil, connectTimeout, requestTimeout)) var o *clusterinfo.ChannelStats for i := 0; !countNum.isSet || countNum.value >= i; i++ { var producers clusterinfo.Producers var err error if len(lookupdHTTPAddrs) != 0 { producers, err = ci.GetLookupdTopicProducers(topic, lookupdHTTPAddrs) } else { producers, err = ci.GetNSQDTopicProducers(topic, nsqdHTTPAddrs) } if err != nil { log.Fatalf("ERROR: failed to get topic producers - %s", err) } _, channelStats, err := ci.GetNSQDStats(producers, topic, channel, false) if err != nil { log.Fatalf("ERROR: failed to get nsqd stats - %s", err) } c, ok := channelStats[channel] if !ok { log.Fatalf("ERROR: failed to find channel(%s) in stats metadata for topic(%s)", channel, topic) } if i%25 == 0 { fmt.Printf("%s+%s+%s\n", "------rate------", "----------------depth----------------", "--------------metadata---------------") fmt.Printf("%7s %7s | %7s %7s %7s %5s %5s | %7s %7s %12s %7s\n", "ingress", "egress", "total", "mem", "disk", "inflt", "def", "req", "t-o", "msgs", "clients") } if o == nil { o = c time.Sleep(interval) continue } // TODO: paused fmt.Printf("%7d %7d | %7d %7d %7d %5d %5d | %7d %7d %12d %7d\n", int64(float64(c.MessageCount-o.MessageCount)/interval.Seconds()), int64(float64(c.MessageCount-o.MessageCount-(c.Depth-o.Depth))/interval.Seconds()), c.Depth, c.MemoryDepth, c.BackendDepth, c.InFlightCount, c.DeferredCount, c.RequeueCount, c.TimeoutCount, c.MessageCount, c.ClientCount) o = c time.Sleep(interval) } os.Exit(0) } func checkAddrs(addrs []string) error { for _, a := range addrs { if strings.HasPrefix(a, "http") { return errors.New("address should not contain scheme") } } return nil } func main() { flag.Parse() if *showVersion { fmt.Printf("nsq_stat v%s\n", version.Binary) return } if *topic == "" || *channel == "" { log.Fatal("--topic and --channel are required") } intvl := *interval if int64(intvl) <= 0 { log.Fatal("--interval should be positive") } connectTimeout := *httpConnectTimeout if int64(connectTimeout) <= 0 { log.Fatal("--http-client-connect-timeout should be positive") } requestTimeout := *httpRequestTimeout if int64(requestTimeout) <= 0 { log.Fatal("--http-client-request-timeout should be positive") } if countNum.isSet && countNum.value <= 0 { log.Fatal("--count should be positive") } if len(nsqdHTTPAddrs) == 0 && len(lookupdHTTPAddrs) == 0 { log.Fatal("--nsqd-http-address or --lookupd-http-address required") } if len(nsqdHTTPAddrs) > 0 && len(lookupdHTTPAddrs) > 0 { log.Fatal("use --nsqd-http-address or --lookupd-http-address not both") } if err := checkAddrs(nsqdHTTPAddrs); err != nil { log.Fatalf("--nsqd-http-address error - %s", err) } if err := checkAddrs(lookupdHTTPAddrs); err != nil { log.Fatalf("--lookupd-http-address error - %s", err) } termChan := make(chan os.Signal, 1) signal.Notify(termChan, syscall.SIGHUP, syscall.SIGINT, syscall.SIGTERM) go statLoop(intvl, connectTimeout, requestTimeout, *topic, *channel, nsqdHTTPAddrs, lookupdHTTPAddrs) <-termChan } ================================================ FILE: apps/nsq_tail/nsq_tail.go ================================================ package main import ( "flag" "fmt" "log" "math/rand" "os" "os/signal" "syscall" "time" "github.com/nsqio/go-nsq" "github.com/nsqio/nsq/internal/app" "github.com/nsqio/nsq/internal/version" ) var ( showVersion = flag.Bool("version", false, "print version string") channel = flag.String("channel", "", "NSQ channel") maxInFlight = flag.Int("max-in-flight", 200, "max number of messages to allow in flight") totalMessages = flag.Int("n", 0, "total messages to show (will wait if starved)") printTopic = flag.Bool("print-topic", false, "print topic name where message was received") nsqdTCPAddrs = app.StringArray{} lookupdHTTPAddrs = app.StringArray{} topics = app.StringArray{} ) func init() { flag.Var(&nsqdTCPAddrs, "nsqd-tcp-address", "nsqd TCP address (may be given multiple times)") flag.Var(&lookupdHTTPAddrs, "lookupd-http-address", "lookupd HTTP address (may be given multiple times)") flag.Var(&topics, "topic", "NSQ topic (may be given multiple times)") } type TailHandler struct { topicName string totalMessages int messagesShown int } func (th *TailHandler) HandleMessage(m *nsq.Message) error { th.messagesShown++ if *printTopic { _, err := os.Stdout.WriteString(th.topicName) if err != nil { log.Fatalf("ERROR: failed to write to os.Stdout - %s", err) } _, err = os.Stdout.WriteString(" | ") if err != nil { log.Fatalf("ERROR: failed to write to os.Stdout - %s", err) } } _, err := os.Stdout.Write(m.Body) if err != nil { log.Fatalf("ERROR: failed to write to os.Stdout - %s", err) } _, err = os.Stdout.WriteString("\n") if err != nil { log.Fatalf("ERROR: failed to write to os.Stdout - %s", err) } if th.totalMessages > 0 && th.messagesShown >= th.totalMessages { os.Exit(0) } return nil } func main() { cfg := nsq.NewConfig() flag.Var(&nsq.ConfigFlag{cfg}, "consumer-opt", "option to passthrough to nsq.Consumer (may be given multiple times, http://godoc.org/github.com/nsqio/go-nsq#Config)") flag.Parse() if *showVersion { fmt.Printf("nsq_tail v%s\n", version.Binary) return } if *channel == "" { rand.Seed(time.Now().UnixNano()) *channel = fmt.Sprintf("tail%06d#ephemeral", rand.Int()%999999) } if len(nsqdTCPAddrs) == 0 && len(lookupdHTTPAddrs) == 0 { log.Fatal("--nsqd-tcp-address or --lookupd-http-address required") } if len(nsqdTCPAddrs) > 0 && len(lookupdHTTPAddrs) > 0 { log.Fatal("use --nsqd-tcp-address or --lookupd-http-address not both") } if len(topics) == 0 { log.Fatal("--topic required") } sigChan := make(chan os.Signal, 1) signal.Notify(sigChan, syscall.SIGINT, syscall.SIGTERM) // Don't ask for more messages than we want if *totalMessages > 0 && *totalMessages < *maxInFlight { *maxInFlight = *totalMessages } cfg.UserAgent = fmt.Sprintf("nsq_tail/%s go-nsq/%s", version.Binary, nsq.VERSION) cfg.MaxInFlight = *maxInFlight consumers := []*nsq.Consumer{} for i := 0; i < len(topics); i++ { log.Printf("Adding consumer for topic: %s\n", topics[i]) consumer, err := nsq.NewConsumer(topics[i], *channel, cfg) if err != nil { log.Fatal(err) } consumer.AddHandler(&TailHandler{topicName: topics[i], totalMessages: *totalMessages}) err = consumer.ConnectToNSQDs(nsqdTCPAddrs) if err != nil { log.Fatal(err) } err = consumer.ConnectToNSQLookupds(lookupdHTTPAddrs) if err != nil { log.Fatal(err) } consumers = append(consumers, consumer) } <-sigChan for _, consumer := range consumers { consumer.Stop() } for _, consumer := range consumers { <-consumer.StopChan } } ================================================ FILE: apps/nsq_to_file/file_logger.go ================================================ package main import ( "compress/gzip" "errors" "fmt" "io" "os" "path" "path/filepath" "strings" "time" "github.com/nsqio/go-nsq" "github.com/nsqio/nsq/internal/lg" ) type FileLogger struct { logf lg.AppLogFunc opts *Options topic string consumer *nsq.Consumer out *os.File writer io.Writer gzipWriter *gzip.Writer logChan chan *nsq.Message filenameFormat string termChan chan bool hupChan chan bool // for rotation filename string openTime time.Time filesize int64 rev uint } func NewFileLogger(logf lg.AppLogFunc, opts *Options, topic string, cfg *nsq.Config) (*FileLogger, error) { computedFilenameFormat, err := computeFilenameFormat(opts, topic) if err != nil { return nil, err } consumer, err := nsq.NewConsumer(topic, opts.Channel, cfg) if err != nil { return nil, err } f := &FileLogger{ logf: logf, opts: opts, topic: topic, consumer: consumer, logChan: make(chan *nsq.Message, 1), filenameFormat: computedFilenameFormat, termChan: make(chan bool), hupChan: make(chan bool), } consumer.AddHandler(f) err = consumer.ConnectToNSQDs(opts.NSQDTCPAddrs) if err != nil { return nil, err } err = consumer.ConnectToNSQLookupds(opts.NSQLookupdHTTPAddrs) if err != nil { return nil, err } return f, nil } func (f *FileLogger) HandleMessage(m *nsq.Message) error { m.DisableAutoResponse() f.logChan <- m return nil } func (f *FileLogger) router() { pos := 0 output := make([]*nsq.Message, f.opts.MaxInFlight) sync := false ticker := time.NewTicker(f.opts.SyncInterval) closeFile := false exit := false for { select { case <-f.consumer.StopChan: sync = true closeFile = true exit = true case <-f.termChan: ticker.Stop() f.consumer.Stop() sync = true case <-f.hupChan: sync = true closeFile = true case <-ticker.C: if f.needsRotation() { if f.opts.SkipEmptyFiles { closeFile = true } else { f.updateFile() } } sync = true case m := <-f.logChan: if f.needsRotation() { f.updateFile() sync = true } _, err := f.Write(m.Body) if err != nil { f.logf(lg.FATAL, "[%s/%s] writing message to disk: %s", f.topic, f.opts.Channel, err) os.Exit(1) } _, err = f.Write([]byte("\n")) if err != nil { f.logf(lg.FATAL, "[%s/%s] writing newline to disk: %s", f.topic, f.opts.Channel, err) os.Exit(1) } output[pos] = m pos++ if pos == cap(output) { sync = true } } if sync || f.consumer.IsStarved() { if pos > 0 { f.logf(lg.INFO, "[%s/%s] syncing %d records to disk", f.topic, f.opts.Channel, pos) err := f.Sync() if err != nil { f.logf(lg.FATAL, "[%s/%s] failed syncing messages: %s", f.topic, f.opts.Channel, err) os.Exit(1) } for pos > 0 { pos-- m := output[pos] m.Finish() output[pos] = nil } } sync = false } if closeFile { f.Close() closeFile = false } if exit { break } } } func (f *FileLogger) Close() { if f.out == nil { return } if f.gzipWriter != nil { err := f.gzipWriter.Close() if err != nil { f.logf(lg.FATAL, "[%s/%s] failed to close GZIP writer: %s", f.topic, f.opts.Channel, err) os.Exit(1) } } err := f.out.Sync() if err != nil { f.logf(lg.FATAL, "[%s/%s] failed to fsync output file: %s", f.topic, f.opts.Channel, err) os.Exit(1) } err = f.out.Close() if err != nil { f.logf(lg.FATAL, "[%s/%s] failed to close output file: %s", f.topic, f.opts.Channel, err) os.Exit(1) } // Move file from work dir to output dir if necessary, taking care not // to overwrite existing files if f.opts.WorkDir != f.opts.OutputDir { src := f.out.Name() dst := filepath.Join(f.opts.OutputDir, strings.TrimPrefix(src, f.opts.WorkDir)) // Optimistic rename f.logf(lg.INFO, "[%s/%s] moving finished file %s to %s", f.topic, f.opts.Channel, src, dst) err := exclusiveRename(src, dst) if err == nil { return } else if !os.IsExist(err) { f.logf(lg.FATAL, "[%s/%s] unable to move file from %s to %s: %s", f.topic, f.opts.Channel, src, dst, err) os.Exit(1) } // Optimistic rename failed, so we need to generate a new // destination file name by bumping the revision number. _, filenameTmpl := filepath.Split(f.filename) dstDir, _ := filepath.Split(dst) dstTmpl := filepath.Join(dstDir, filenameTmpl) for i := f.rev + 1; ; i++ { f.logf(lg.WARN, "[%s/%s] destination file already exists: %s", f.topic, f.opts.Channel, dst) dst := strings.Replace(dstTmpl, "", fmt.Sprintf("-%06d", i), -1) err := exclusiveRename(src, dst) if err != nil { if os.IsExist(err) { continue // next rev } f.logf(lg.FATAL, "[%s/%s] unable to rename file from %s to %s: %s", f.topic, f.opts.Channel, src, dst, err) os.Exit(1) } f.logf(lg.INFO, "[%s/%s] renamed finished file %s to %s to avoid overwrite", f.topic, f.opts.Channel, src, dst) break } } f.out = nil } func (f *FileLogger) Write(p []byte) (int, error) { n, err := f.writer.Write(p) f.filesize += int64(n) return n, err } func (f *FileLogger) Sync() error { var err error if f.gzipWriter != nil { // finish current gzip stream and start a new one (concatenated) // gzip stream trailer has checksum, and can indicate which messages were ACKed err = f.gzipWriter.Close() if err != nil { return err } err = f.out.Sync() f.gzipWriter, _ = gzip.NewWriterLevel(f.out, f.opts.GZIPLevel) f.writer = f.gzipWriter } else { err = f.out.Sync() } return err } func (f *FileLogger) currentFilename() string { t := time.Now() datetime := strftime(f.opts.DatetimeFormat, t) return strings.Replace(f.filenameFormat, "", datetime, -1) } func (f *FileLogger) needsRotation() bool { if f.out == nil { return true } filename := f.currentFilename() if filename != f.filename { f.logf(lg.INFO, "[%s/%s] new filename %s, rotating...", f.topic, f.opts.Channel, filename) return true // rotate by filename } if f.opts.RotateInterval > 0 { if s := time.Since(f.openTime); s > f.opts.RotateInterval { f.logf(lg.INFO, "[%s/%s] %s since last open, rotating...", f.topic, f.opts.Channel, s) return true // rotate by interval } } if f.opts.RotateSize > 0 && f.filesize > f.opts.RotateSize { f.logf(lg.INFO, "[%s/%s] %s currently %d bytes (> %d), rotating...", f.topic, f.opts.Channel, f.out.Name(), f.filesize, f.opts.RotateSize) return true // rotate by size } return false } func (f *FileLogger) updateFile() { f.Close() // uses current f.filename and f.rev to resolve rename dst conflict filename := f.currentFilename() if filename != f.filename { f.rev = 0 // reset revision to 0 if it is a new filename } else { f.rev++ } f.filename = filename f.openTime = time.Now() fullPath := path.Join(f.opts.WorkDir, filename) err := makeDirFromPath(f.logf, fullPath) if err != nil { f.logf(lg.FATAL, "[%s/%s] unable to create dir: %s", f.topic, f.opts.Channel, err) os.Exit(1) } var fi os.FileInfo for ; ; f.rev++ { absFilename := strings.Replace(fullPath, "", fmt.Sprintf("-%06d", f.rev), -1) // If we're using a working directory for in-progress files, // proactively check for duplicate file names in the output dir to // prevent conflicts on rename in the normal case if f.opts.WorkDir != f.opts.OutputDir { outputFileName := filepath.Join(f.opts.OutputDir, strings.TrimPrefix(absFilename, f.opts.WorkDir)) err := makeDirFromPath(f.logf, outputFileName) if err != nil { f.logf(lg.FATAL, "[%s/%s] unable to create dir: %s", f.topic, f.opts.Channel, err) os.Exit(1) } _, err = os.Stat(outputFileName) if err == nil { f.logf(lg.WARN, "[%s/%s] output file already exists: %s", f.topic, f.opts.Channel, outputFileName) continue // next rev } else if !os.IsNotExist(err) { f.logf(lg.FATAL, "[%s/%s] unable to stat output file %s: %s", f.topic, f.opts.Channel, outputFileName, err) os.Exit(1) } } openFlag := os.O_WRONLY | os.O_CREATE if f.opts.GZIP || f.opts.RotateInterval > 0 { openFlag |= os.O_EXCL } else { openFlag |= os.O_APPEND } f.out, err = os.OpenFile(absFilename, openFlag, 0666) if err != nil { if os.IsExist(err) { f.logf(lg.WARN, "[%s/%s] working file already exists: %s", f.topic, f.opts.Channel, absFilename) continue // next rev } f.logf(lg.FATAL, "[%s/%s] unable to open %s: %s", f.topic, f.opts.Channel, absFilename, err) os.Exit(1) } f.logf(lg.INFO, "[%s/%s] opening %s", f.topic, f.opts.Channel, absFilename) fi, err = f.out.Stat() if err != nil { f.logf(lg.FATAL, "[%s/%s] unable to stat file %s: %s", f.topic, f.opts.Channel, f.out.Name(), err) } f.filesize = fi.Size() if f.opts.RotateSize > 0 && f.filesize > f.opts.RotateSize { f.logf(lg.INFO, "[%s/%s] %s currently %d bytes (> %d), rotating...", f.topic, f.opts.Channel, f.out.Name(), f.filesize, f.opts.RotateSize) continue // next rev } break // good file } if f.opts.GZIP { f.gzipWriter, _ = gzip.NewWriterLevel(f.out, f.opts.GZIPLevel) f.writer = f.gzipWriter } else { f.writer = f.out } } func makeDirFromPath(logf lg.AppLogFunc, path string) error { dir, _ := filepath.Split(path) if dir != "" { return os.MkdirAll(dir, 0770) } return nil } func exclusiveRename(src, dst string) error { err := os.Link(src, dst) if err != nil { return err } err = os.Remove(src) if err != nil { return err } return nil } func computeFilenameFormat(opts *Options, topic string) (string, error) { hostname, err := os.Hostname() if err != nil { return "", err } shortHostname := strings.Split(hostname, ".")[0] identifier := shortHostname if len(opts.HostIdentifier) != 0 { identifier = strings.Replace(opts.HostIdentifier, "", shortHostname, -1) identifier = strings.Replace(identifier, "", hostname, -1) } cff := opts.FilenameFormat if opts.GZIP || opts.RotateSize > 0 || opts.RotateInterval > 0 || opts.WorkDir != opts.OutputDir { if !strings.Contains(cff, "") { return "", errors.New("missing in --filename-format when gzip, rotation, or work dir enabled") } } else { // remove as we don't need it cff = strings.Replace(cff, "", "", -1) } cff = strings.Replace(cff, "", topic, -1) cff = strings.Replace(cff, "", identifier, -1) cff = strings.Replace(cff, "", fmt.Sprintf("%d", os.Getpid()), -1) if opts.GZIP && !strings.HasSuffix(cff, ".gz") { cff = cff + ".gz" } return cff, nil } ================================================ FILE: apps/nsq_to_file/nsq_to_file.go ================================================ // This is a client that writes out to a file, and optionally rolls the file package main import ( "flag" "fmt" "log" "os" "os/signal" "syscall" "time" "github.com/mreiferson/go-options" "github.com/nsqio/go-nsq" "github.com/nsqio/nsq/internal/app" "github.com/nsqio/nsq/internal/lg" "github.com/nsqio/nsq/internal/version" ) func flagSet() *flag.FlagSet { fs := flag.NewFlagSet("nsqd", flag.ExitOnError) fs.Bool("version", false, "print version string") fs.String("log-level", "info", "set log verbosity: debug, info, warn, error, or fatal") fs.String("log-prefix", "[nsq_to_file] ", "log message prefix") fs.String("channel", "nsq_to_file", "nsq channel") fs.Int("max-in-flight", 200, "max number of messages to allow in flight") fs.String("output-dir", "/tmp", "directory to write output files to") fs.String("work-dir", "", "directory for in-progress files before moving to output-dir") fs.String("datetime-format", "%Y-%m-%d_%H", "strftime compatible format for in filename format") fs.String("filename-format", "...log", "output filename format (, , , , are replaced. is increased when file already exists)") fs.String("host-identifier", "", "value to output in log filename in place of hostname. and are valid replacement tokens") fs.Int("gzip-level", 6, "gzip compression level (1-9, 1=BestSpeed, 9=BestCompression)") fs.Bool("gzip", false, "gzip output files.") fs.Bool("skip-empty-files", false, "skip writing empty files") fs.Duration("topic-refresh", time.Minute, "how frequently the topic list should be refreshed") fs.String("topic-pattern", "", "only log topics matching the following pattern") fs.Int64("rotate-size", 0, "rotate the file when it grows bigger than `rotate-size` bytes") fs.Duration("rotate-interval", 0, "rotate the file every duration") fs.Duration("sync-interval", 30*time.Second, "sync file to disk every duration") fs.Duration("http-client-connect-timeout", 2*time.Second, "timeout for HTTP connect") fs.Duration("http-client-request-timeout", 5*time.Second, "timeout for HTTP request") nsqdTCPAddrs := app.StringArray{} lookupdHTTPAddrs := app.StringArray{} topics := app.StringArray{} consumerOpts := app.StringArray{} fs.Var(&nsqdTCPAddrs, "nsqd-tcp-address", "nsqd TCP address (may be given multiple times)") fs.Var(&lookupdHTTPAddrs, "lookupd-http-address", "lookupd HTTP address (may be given multiple times)") fs.Var(&topics, "topic", "nsq topic (may be given multiple times)") fs.Var(&consumerOpts, "consumer-opt", "option to passthrough to nsq.Consumer (may be given multiple times, http://godoc.org/github.com/nsqio/go-nsq#Config)") return fs } func main() { fs := flagSet() fs.Parse(os.Args[1:]) if args := fs.Args(); len(args) > 0 { log.Fatalf("unknown arguments: %s", args) } opts := NewOptions() options.Resolve(opts, fs, nil) logger := log.New(os.Stderr, opts.LogPrefix, log.Ldate|log.Ltime|log.Lmicroseconds) logLevel, err := lg.ParseLogLevel(opts.LogLevel) if err != nil { log.Fatal("--log-level is invalid") } logf := func(lvl lg.LogLevel, f string, args ...interface{}) { lg.Logf(logger, logLevel, lvl, f, args...) } if fs.Lookup("version").Value.(flag.Getter).Get().(bool) { fmt.Printf("nsq_to_file v%s\n", version.Binary) return } if opts.Channel == "" { log.Fatal("--channel is required") } if opts.HTTPClientConnectTimeout <= 0 { log.Fatal("--http-client-connect-timeout should be positive") } if opts.HTTPClientRequestTimeout <= 0 { log.Fatal("--http-client-request-timeout should be positive") } if len(opts.NSQDTCPAddrs) == 0 && len(opts.NSQLookupdHTTPAddrs) == 0 { log.Fatal("--nsqd-tcp-address or --lookupd-http-address required.") } if len(opts.NSQDTCPAddrs) != 0 && len(opts.NSQLookupdHTTPAddrs) != 0 { log.Fatal("use --nsqd-tcp-address or --lookupd-http-address not both") } if opts.GZIPLevel < 1 || opts.GZIPLevel > 9 { log.Fatalf("invalid --gzip-level value (%d), should be 1-9", opts.GZIPLevel) } if len(opts.Topics) == 0 && len(opts.TopicPattern) == 0 { log.Fatal("--topic or --topic-pattern required") } if len(opts.Topics) == 0 && len(opts.NSQLookupdHTTPAddrs) == 0 { log.Fatal("--lookupd-http-address must be specified when no --topic specified") } if opts.WorkDir == "" { opts.WorkDir = opts.OutputDir } cfg := nsq.NewConfig() cfgFlag := nsq.ConfigFlag{cfg} for _, opt := range opts.ConsumerOpts { cfgFlag.Set(opt) } cfg.UserAgent = fmt.Sprintf("nsq_to_file/%s go-nsq/%s", version.Binary, nsq.VERSION) cfg.MaxInFlight = opts.MaxInFlight hupChan := make(chan os.Signal, 1) termChan := make(chan os.Signal, 1) signal.Notify(hupChan, syscall.SIGHUP) signal.Notify(termChan, syscall.SIGINT, syscall.SIGTERM) discoverer := newTopicDiscoverer(logf, opts, cfg, hupChan, termChan) discoverer.run() } ================================================ FILE: apps/nsq_to_file/options.go ================================================ package main import "time" type Options struct { Topics []string `flag:"topic"` TopicPattern string `flag:"topic-pattern"` TopicRefreshInterval time.Duration `flag:"topic-refresh"` Channel string `flag:"channel"` NSQDTCPAddrs []string `flag:"nsqd-tcp-address"` NSQLookupdHTTPAddrs []string `flag:"lookupd-http-address"` ConsumerOpts []string `flag:"consumer-opt"` MaxInFlight int `flag:"max-in-flight"` HTTPClientConnectTimeout time.Duration `flag:"http-client-connect-timeout"` HTTPClientRequestTimeout time.Duration `flag:"http-client-request-timeout"` LogPrefix string `flag:"log-prefix"` LogLevel string `flag:"log-level"` OutputDir string `flag:"output-dir"` WorkDir string `flag:"work-dir"` DatetimeFormat string `flag:"datetime-format"` FilenameFormat string `flag:"filename-format"` HostIdentifier string `flag:"host-identifier"` GZIPLevel int `flag:"gzip-level"` GZIP bool `flag:"gzip"` SkipEmptyFiles bool `flag:"skip-empty-files"` RotateSize int64 `flag:"rotate-size"` RotateInterval time.Duration `flag:"rotate-interval"` SyncInterval time.Duration `flag:"sync-interval"` } func NewOptions() *Options { return &Options{ LogPrefix: "[nsq_to_file] ", LogLevel: "info", Channel: "nsq_to_file", MaxInFlight: 200, OutputDir: "/tmp", DatetimeFormat: "%Y-%m-%d_%H", FilenameFormat: "...log", GZIPLevel: 6, TopicRefreshInterval: time.Minute, SyncInterval: 30 * time.Second, HTTPClientConnectTimeout: 2 * time.Second, HTTPClientRequestTimeout: 5 * time.Second, } } ================================================ FILE: apps/nsq_to_file/strftime.go ================================================ // COPIED FROM https://github.com/jehiah/go-strftime package main import ( "time" ) // taken from time/format.go var conversion = map[string]string{ /*stdLongMonth */ "B": "January", /*stdMonth */ "b": "Jan", // stdNumMonth */ "m": "1", /*stdZeroMonth */ "m": "01", /*stdLongWeekDay */ "A": "Monday", /*stdWeekDay */ "a": "Mon", // stdDay */ "d": "2", // stdUnderDay */ "d": "_2", /*stdZeroDay */ "d": "02", /*stdHour */ "H": "15", // stdHour12 */ "I": "3", /*stdZeroHour12 */ "I": "03", // stdMinute */ "M": "4", /*stdZeroMinute */ "M": "04", // stdSecond */ "S": "5", /*stdZeroSecond */ "S": "05", /*stdLongYear */ "Y": "2006", /*stdYear */ "y": "06", /*stdPM */ "p": "PM", // stdpm */ "p": "pm", /*stdTZ */ "Z": "MST", // stdISO8601TZ */ "z": "Z0700", // prints Z for UTC // stdISO8601ColonTZ */ "z": "Z07:00", // prints Z for UTC /*stdNumTZ */ "z": "-0700", // always numeric // stdNumShortTZ */ "b": "-07", // always numeric // stdNumColonTZ */ "b": "-07:00", // always numeric "%": "%", } // This is an alternative to time.Format because no one knows // what date 040305 is supposed to create when used as a 'layout' string // this takes standard strftime format options. For a complete list // of format options see http://strftime.org/ func strftime(format string, t time.Time) string { layout := "" length := len(format) for i := 0; i < length; i++ { if format[i] == '%' && i <= length-2 { if layoutCmd, ok := conversion[format[i+1:i+2]]; ok { layout = layout + layoutCmd i++ continue } } layout = layout + format[i:i+1] } return t.Format(layout) } ================================================ FILE: apps/nsq_to_file/topic_discoverer.go ================================================ package main import ( "os" "regexp" "sync" "time" "github.com/nsqio/go-nsq" "github.com/nsqio/nsq/internal/clusterinfo" "github.com/nsqio/nsq/internal/http_api" "github.com/nsqio/nsq/internal/lg" ) type TopicDiscoverer struct { logf lg.AppLogFunc opts *Options ci *clusterinfo.ClusterInfo topics map[string]*FileLogger hupChan chan os.Signal termChan chan os.Signal wg sync.WaitGroup cfg *nsq.Config } func newTopicDiscoverer(logf lg.AppLogFunc, opts *Options, cfg *nsq.Config, hupChan chan os.Signal, termChan chan os.Signal) *TopicDiscoverer { client := http_api.NewClient(nil, opts.HTTPClientConnectTimeout, opts.HTTPClientRequestTimeout) return &TopicDiscoverer{ logf: logf, opts: opts, ci: clusterinfo.New(nil, client), topics: make(map[string]*FileLogger), hupChan: hupChan, termChan: termChan, cfg: cfg, } } func (t *TopicDiscoverer) updateTopics(topics []string) { for _, topic := range topics { if _, ok := t.topics[topic]; ok { continue } if !t.isTopicAllowed(topic) { t.logf(lg.WARN, "skipping topic %s (doesn't match pattern %s)", topic, t.opts.TopicPattern) continue } fl, err := NewFileLogger(t.logf, t.opts, topic, t.cfg) if err != nil { t.logf(lg.ERROR, "couldn't create logger for new topic %s: %s", topic, err) continue } t.topics[topic] = fl t.wg.Add(1) go func(fl *FileLogger) { fl.router() t.wg.Done() }(fl) } } func (t *TopicDiscoverer) run() { var ticker <-chan time.Time if len(t.opts.Topics) == 0 { ticker = time.Tick(t.opts.TopicRefreshInterval) } t.updateTopics(t.opts.Topics) forloop: for { select { case <-ticker: newTopics, err := t.ci.GetLookupdTopics(t.opts.NSQLookupdHTTPAddrs) if err != nil { t.logf(lg.ERROR, "could not retrieve topic list: %s", err) continue } t.updateTopics(newTopics) case <-t.termChan: for _, fl := range t.topics { close(fl.termChan) } break forloop case <-t.hupChan: for _, fl := range t.topics { fl.hupChan <- true } } } t.wg.Wait() } func (t *TopicDiscoverer) isTopicAllowed(topic string) bool { if t.opts.TopicPattern == "" { return true } match, err := regexp.MatchString(t.opts.TopicPattern, topic) if err != nil { return false } return match } ================================================ FILE: apps/nsq_to_http/http.go ================================================ package main import ( "bytes" "fmt" "net/http" "github.com/nsqio/nsq/internal/version" ) var httpclient *http.Client var userAgent string func init() { userAgent = fmt.Sprintf("nsq_to_http v%s", version.Binary) } func HTTPGet(endpoint string) (*http.Response, error) { req, err := http.NewRequest("GET", endpoint, nil) if err != nil { return nil, err } req.Header.Set("User-Agent", userAgent) for key, val := range validCustomHeaders { req.Header.Set(key, val) } return httpclient.Do(req) } func HTTPPost(endpoint string, body *bytes.Buffer) (*http.Response, error) { req, err := http.NewRequest("POST", endpoint, body) if err != nil { return nil, err } req.Header.Set("User-Agent", userAgent) req.Header.Set("Content-Type", *contentType) for key, val := range validCustomHeaders { req.Header.Set(key, val) } return httpclient.Do(req) } ================================================ FILE: apps/nsq_to_http/nsq_to_http.go ================================================ // This is an NSQ client that reads the specified topic/channel // and performs HTTP requests (GET/POST) to the specified endpoints package main import ( "bytes" "flag" "fmt" "io" "log" "math/rand" "net/http" "net/url" "os" "os/signal" "strings" "sync/atomic" "syscall" "time" "github.com/bitly/go-hostpool" "github.com/bitly/timer_metrics" "github.com/nsqio/go-nsq" "github.com/nsqio/nsq/internal/app" "github.com/nsqio/nsq/internal/http_api" "github.com/nsqio/nsq/internal/version" ) const ( ModeAll = iota ModeRoundRobin ModeHostPool ) var ( showVersion = flag.Bool("version", false, "print version string") topic = flag.String("topic", "", "nsq topic") channel = flag.String("channel", "nsq_to_http", "nsq channel") maxInFlight = flag.Int("max-in-flight", 200, "max number of messages to allow in flight") numPublishers = flag.Int("n", 100, "number of concurrent publishers") mode = flag.String("mode", "hostpool", "the upstream request mode options: round-robin, hostpool (default), epsilon-greedy") sample = flag.Float64("sample", 1.0, "% of messages to publish (float b/w 0 -> 1)") httpConnectTimeout = flag.Duration("http-client-connect-timeout", 2*time.Second, "timeout for HTTP connect") httpRequestTimeout = flag.Duration("http-client-request-timeout", 20*time.Second, "timeout for HTTP request") statusEvery = flag.Int("status-every", 250, "the # of requests between logging status (per handler), 0 disables") contentType = flag.String("content-type", "application/octet-stream", "the Content-Type used for POST requests") getAddrs = app.StringArray{} postAddrs = app.StringArray{} customHeaders = app.StringArray{} nsqdTCPAddrs = app.StringArray{} lookupdHTTPAddrs = app.StringArray{} validCustomHeaders map[string]string ) func init() { flag.Var(&postAddrs, "post", "HTTP address to make a POST request to. data will be in the body (may be given multiple times)") flag.Var(&customHeaders, "header", "Custom header for HTTP requests (may be given multiple times)") flag.Var(&getAddrs, "get", "HTTP address to make a GET request to. '%s' will be printf replaced with data (may be given multiple times)") flag.Var(&nsqdTCPAddrs, "nsqd-tcp-address", "nsqd TCP address (may be given multiple times)") flag.Var(&lookupdHTTPAddrs, "lookupd-http-address", "lookupd HTTP address (may be given multiple times)") } type Publisher interface { Publish(string, []byte) error } type PublishHandler struct { // 64bit atomic vars need to be first for proper alignment on 32bit platforms counter uint64 Publisher addresses app.StringArray mode int hostPool hostpool.HostPool perAddressStatus map[string]*timer_metrics.TimerMetrics timermetrics *timer_metrics.TimerMetrics } func (ph *PublishHandler) HandleMessage(m *nsq.Message) error { if *sample < 1.0 && rand.Float64() > *sample { return nil } startTime := time.Now() switch ph.mode { case ModeAll: for _, addr := range ph.addresses { st := time.Now() err := ph.Publish(addr, m.Body) if err != nil { return err } ph.perAddressStatus[addr].Status(st) } case ModeRoundRobin: counter := atomic.AddUint64(&ph.counter, 1) idx := counter % uint64(len(ph.addresses)) addr := ph.addresses[idx] err := ph.Publish(addr, m.Body) if err != nil { return err } ph.perAddressStatus[addr].Status(startTime) case ModeHostPool: hostPoolResponse := ph.hostPool.Get() addr := hostPoolResponse.Host() err := ph.Publish(addr, m.Body) hostPoolResponse.Mark(err) if err != nil { return err } ph.perAddressStatus[addr].Status(startTime) } ph.timermetrics.Status(startTime) return nil } type PostPublisher struct{} func (p *PostPublisher) Publish(addr string, msg []byte) error { buf := bytes.NewBuffer(msg) resp, err := HTTPPost(addr, buf) if err != nil { return err } io.Copy(io.Discard, resp.Body) resp.Body.Close() if resp.StatusCode < 200 || resp.StatusCode >= 300 { return fmt.Errorf("got status code %d", resp.StatusCode) } return nil } type GetPublisher struct{} func (p *GetPublisher) Publish(addr string, msg []byte) error { endpoint := fmt.Sprintf(addr, url.QueryEscape(string(msg))) resp, err := HTTPGet(endpoint) if err != nil { return err } io.Copy(io.Discard, resp.Body) resp.Body.Close() if resp.StatusCode != 200 { return fmt.Errorf("got status code %d", resp.StatusCode) } return nil } func main() { var publisher Publisher var addresses app.StringArray var selectedMode int cfg := nsq.NewConfig() flag.Var(&nsq.ConfigFlag{cfg}, "consumer-opt", "option to passthrough to nsq.Consumer (may be given multiple times, http://godoc.org/github.com/nsqio/go-nsq#Config)") flag.Parse() httpclient = &http.Client{Transport: http_api.NewDeadlineTransport(*httpConnectTimeout, *httpRequestTimeout), Timeout: *httpRequestTimeout} if *showVersion { fmt.Printf("nsq_to_http v%s\n", version.Binary) return } if len(customHeaders) > 0 { var err error validCustomHeaders, err = parseCustomHeaders(customHeaders) if err != nil { log.Fatal("--header value format should be 'key=value'") } } if *topic == "" || *channel == "" { log.Fatal("--topic and --channel are required") } if *contentType != flag.Lookup("content-type").DefValue { if len(postAddrs) == 0 { log.Fatal("--content-type only used with --post") } if len(*contentType) == 0 { log.Fatal("--content-type requires a value when used") } } if len(nsqdTCPAddrs) == 0 && len(lookupdHTTPAddrs) == 0 { log.Fatal("--nsqd-tcp-address or --lookupd-http-address required") } if len(nsqdTCPAddrs) > 0 && len(lookupdHTTPAddrs) > 0 { log.Fatal("use --nsqd-tcp-address or --lookupd-http-address not both") } if len(getAddrs) == 0 && len(postAddrs) == 0 { log.Fatal("--get or --post required") } if len(getAddrs) > 0 && len(postAddrs) > 0 { log.Fatal("use --get or --post not both") } if len(getAddrs) > 0 { for _, get := range getAddrs { if strings.Count(get, "%s") != 1 { log.Fatal("invalid GET address - must be a printf string") } } } switch *mode { case "round-robin": selectedMode = ModeRoundRobin case "hostpool", "epsilon-greedy": selectedMode = ModeHostPool } if *sample > 1.0 || *sample < 0.0 { log.Fatal("ERROR: --sample must be between 0.0 and 1.0") } termChan := make(chan os.Signal, 1) signal.Notify(termChan, syscall.SIGINT, syscall.SIGTERM) if len(postAddrs) > 0 { publisher = &PostPublisher{} addresses = postAddrs } else { publisher = &GetPublisher{} addresses = getAddrs } cfg.UserAgent = fmt.Sprintf("nsq_to_http/%s go-nsq/%s", version.Binary, nsq.VERSION) cfg.MaxInFlight = *maxInFlight consumer, err := nsq.NewConsumer(*topic, *channel, cfg) if err != nil { log.Fatal(err) } perAddressStatus := make(map[string]*timer_metrics.TimerMetrics) if len(addresses) == 1 { // disable since there is only one address perAddressStatus[addresses[0]] = timer_metrics.NewTimerMetrics(0, "") } else { for _, a := range addresses { perAddressStatus[a] = timer_metrics.NewTimerMetrics(*statusEvery, fmt.Sprintf("[%s]:", a)) } } hostPool := hostpool.New(addresses) if *mode == "epsilon-greedy" { hostPool = hostpool.NewEpsilonGreedy(addresses, 0, &hostpool.LinearEpsilonValueCalculator{}) } handler := &PublishHandler{ Publisher: publisher, addresses: addresses, mode: selectedMode, hostPool: hostPool, perAddressStatus: perAddressStatus, timermetrics: timer_metrics.NewTimerMetrics(*statusEvery, "[aggregate]:"), } consumer.AddConcurrentHandlers(handler, *numPublishers) err = consumer.ConnectToNSQDs(nsqdTCPAddrs) if err != nil { log.Fatal(err) } err = consumer.ConnectToNSQLookupds(lookupdHTTPAddrs) if err != nil { log.Fatal(err) } for { select { case <-consumer.StopChan: return case <-termChan: consumer.Stop() } } } func parseCustomHeaders(strs []string) (map[string]string, error) { parsedHeaders := make(map[string]string) for _, s := range strs { sp := strings.SplitN(s, ":", 2) if len(sp) != 2 { return nil, fmt.Errorf("invalid header: %q", s) } key := strings.TrimSpace(sp[0]) val := strings.TrimSpace(sp[1]) if key == "" || val == "" { return nil, fmt.Errorf("invalid header: %q", s) } parsedHeaders[key] = val } return parsedHeaders, nil } ================================================ FILE: apps/nsq_to_http/nsq_to_http_test.go ================================================ // This is an NSQ client that reads the specified topic/channel // and performs HTTP requests (GET/POST) to the specified endpoints package main import ( "reflect" "testing" ) func TestParseCustomHeaders(t *testing.T) { type args struct { strs []string } tests := []struct { name string args args want map[string]string wantErr bool }{ { "Valid Custom Headers", args{[]string{"header1: value1", "header2:value2", "header3:value3", "header4:value4"}}, map[string]string{"header1": "value1", "header2": "value2", "header3": "value3", "header4": "value4"}, false, }, { "Invalid Custom Headers where key is present but no value", args{[]string{"header1:", "header2:value2", "header3: value3", "header4:value4"}}, nil, true, }, { "Invalid Custom Headers where key is not present but value is present", args{[]string{"header1: value1", ": value2", "header3:value3", "header4:value4"}}, nil, true, }, { "Invalid Custom Headers where key and value are not present but ':' is specified", args{[]string{"header1:value1", "header2:value2", ":", "header4:value4"}}, nil, true, }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { got, err := parseCustomHeaders(tt.args.strs) if (err != nil) != tt.wantErr { t.Errorf("parseCustomHeaders() error = %v, wantErr %v", err, tt.wantErr) return } if !reflect.DeepEqual(got, tt.want) { t.Errorf("parseCustomHeaders() = %v, want %v", got, tt.want) } }) } } ================================================ FILE: apps/nsq_to_nsq/nsq_to_nsq.go ================================================ // This is an NSQ client that reads the specified topic/channel // and re-publishes the messages to destination nsqd via TCP package main import ( "encoding/json" "errors" "flag" "fmt" "log" "os" "os/signal" "strconv" "sync/atomic" "syscall" "time" "github.com/bitly/go-hostpool" "github.com/bitly/timer_metrics" "github.com/nsqio/go-nsq" "github.com/nsqio/nsq/internal/app" "github.com/nsqio/nsq/internal/protocol" "github.com/nsqio/nsq/internal/version" ) const ( ModeRoundRobin = iota ModeHostPool ) var ( showVersion = flag.Bool("version", false, "print version string") channel = flag.String("channel", "nsq_to_nsq", "nsq channel") destTopic = flag.String("destination-topic", "", "use this destination topic for all consumed topics (default is consumed topic name)") maxInFlight = flag.Int("max-in-flight", 200, "max number of messages to allow in flight") statusEvery = flag.Int("status-every", 250, "the # of requests between logging status (per destination), 0 disables") mode = flag.String("mode", "hostpool", "the upstream request mode options: round-robin, hostpool (default), epsilon-greedy") nsqdTCPAddrs = app.StringArray{} lookupdHTTPAddrs = app.StringArray{} destNsqdTCPAddrs = app.StringArray{} whitelistJSONFields = app.StringArray{} topics = app.StringArray{} requireJSONField = flag.String("require-json-field", "", "for JSON messages: only pass messages that contain this field") requireJSONValue = flag.String("require-json-value", "", "for JSON messages: only pass messages in which the required field has this value") ) func init() { flag.Var(&nsqdTCPAddrs, "nsqd-tcp-address", "nsqd TCP address (may be given multiple times)") flag.Var(&destNsqdTCPAddrs, "destination-nsqd-tcp-address", "destination nsqd TCP address (may be given multiple times)") flag.Var(&lookupdHTTPAddrs, "lookupd-http-address", "lookupd HTTP address (may be given multiple times)") flag.Var(&topics, "topic", "nsq topic (may be given multiple times)") flag.Var(&whitelistJSONFields, "whitelist-json-field", "for JSON messages: pass this field (may be given multiple times)") } type PublishHandler struct { // 64bit atomic vars need to be first for proper alignment on 32bit platforms counter uint64 addresses app.StringArray producers map[string]*nsq.Producer mode int hostPool hostpool.HostPool respChan chan *nsq.ProducerTransaction requireJSONValueParsed bool requireJSONValueIsNumber bool requireJSONNumber float64 perAddressStatus map[string]*timer_metrics.TimerMetrics timermetrics *timer_metrics.TimerMetrics } type TopicHandler struct { publishHandler *PublishHandler destinationTopic string } func (ph *PublishHandler) responder() { var msg *nsq.Message var startTime time.Time var address string var hostPoolResponse hostpool.HostPoolResponse for t := range ph.respChan { switch ph.mode { case ModeRoundRobin: msg = t.Args[0].(*nsq.Message) startTime = t.Args[1].(time.Time) hostPoolResponse = nil address = t.Args[2].(string) case ModeHostPool: msg = t.Args[0].(*nsq.Message) startTime = t.Args[1].(time.Time) hostPoolResponse = t.Args[2].(hostpool.HostPoolResponse) address = hostPoolResponse.Host() } success := t.Error == nil if hostPoolResponse != nil { if !success { hostPoolResponse.Mark(errors.New("failed")) } else { hostPoolResponse.Mark(nil) } } if success { msg.Finish() } else { msg.Requeue(-1) } ph.perAddressStatus[address].Status(startTime) ph.timermetrics.Status(startTime) } } func (ph *PublishHandler) shouldPassMessage(js map[string]interface{}) (bool, bool) { pass := true backoff := false if *requireJSONField == "" { return pass, backoff } if *requireJSONValue != "" && !ph.requireJSONValueParsed { // cache conversion in case needed while filtering json var err error ph.requireJSONNumber, err = strconv.ParseFloat(*requireJSONValue, 64) ph.requireJSONValueIsNumber = (err == nil) ph.requireJSONValueParsed = true } v, ok := js[*requireJSONField] if !ok { pass = false if *requireJSONValue != "" { log.Printf("ERROR: missing field to check required value") backoff = true } } else if *requireJSONValue != "" { // if command-line argument can't convert to float, then it can't match a number // if it can, also integers (up to 2^53 or so) can be compared as float64 if s, ok := v.(string); ok { if s != *requireJSONValue { pass = false } } else if ph.requireJSONValueIsNumber { f, ok := v.(float64) if !ok || f != ph.requireJSONNumber { pass = false } } else { // json value wasn't a plain string, and argument wasn't a number // give up on comparisons of other types pass = false } } return pass, backoff } func filterMessage(js map[string]interface{}, rawMsg []byte) ([]byte, error) { if len(whitelistJSONFields) == 0 { // no change return rawMsg, nil } newMsg := make(map[string]interface{}, len(whitelistJSONFields)) for _, key := range whitelistJSONFields { value, ok := js[key] if ok { // avoid printing int as float (go 1.0) switch tvalue := value.(type) { case float64: ivalue := int64(tvalue) if float64(ivalue) == tvalue { newMsg[key] = ivalue } else { newMsg[key] = tvalue } default: newMsg[key] = value } } } newRawMsg, err := json.Marshal(newMsg) if err != nil { return nil, fmt.Errorf("unable to marshal filtered message %v", newMsg) } return newRawMsg, nil } func (t *TopicHandler) HandleMessage(m *nsq.Message) error { return t.publishHandler.HandleMessage(m, t.destinationTopic) } func (ph *PublishHandler) HandleMessage(m *nsq.Message, destinationTopic string) error { var err error msgBody := m.Body if *requireJSONField != "" || len(whitelistJSONFields) > 0 { var js map[string]interface{} err = json.Unmarshal(msgBody, &js) if err != nil { log.Printf("ERROR: Unable to decode json: %s", msgBody) return nil } if pass, backoff := ph.shouldPassMessage(js); !pass { if backoff { return errors.New("backoff") } return nil } msgBody, err = filterMessage(js, msgBody) if err != nil { log.Printf("ERROR: filterMessage() failed: %s", err) return err } } startTime := time.Now() switch ph.mode { case ModeRoundRobin: counter := atomic.AddUint64(&ph.counter, 1) idx := counter % uint64(len(ph.addresses)) addr := ph.addresses[idx] p := ph.producers[addr] err = p.PublishAsync(destinationTopic, msgBody, ph.respChan, m, startTime, addr) case ModeHostPool: hostPoolResponse := ph.hostPool.Get() p := ph.producers[hostPoolResponse.Host()] err = p.PublishAsync(destinationTopic, msgBody, ph.respChan, m, startTime, hostPoolResponse) if err != nil { hostPoolResponse.Mark(err) } } if err != nil { return err } m.DisableAutoResponse() return nil } func main() { var selectedMode int cCfg := nsq.NewConfig() pCfg := nsq.NewConfig() flag.Var(&nsq.ConfigFlag{cCfg}, "consumer-opt", "option to passthrough to nsq.Consumer (may be given multiple times, see http://godoc.org/github.com/nsqio/go-nsq#Config)") flag.Var(&nsq.ConfigFlag{pCfg}, "producer-opt", "option to passthrough to nsq.Producer (may be given multiple times, see http://godoc.org/github.com/nsqio/go-nsq#Config)") flag.Parse() if *showVersion { fmt.Printf("nsq_to_nsq v%s\n", version.Binary) return } if len(topics) == 0 || *channel == "" { log.Fatal("--topic and --channel are required") } for _, topic := range topics { if !protocol.IsValidTopicName(topic) { log.Fatal("--topic is invalid") } } if *destTopic != "" && !protocol.IsValidTopicName(*destTopic) { log.Fatal("--destination-topic is invalid") } if !protocol.IsValidChannelName(*channel) { log.Fatal("--channel is invalid") } if len(nsqdTCPAddrs) == 0 && len(lookupdHTTPAddrs) == 0 { log.Fatal("--nsqd-tcp-address or --lookupd-http-address required") } if len(nsqdTCPAddrs) > 0 && len(lookupdHTTPAddrs) > 0 { log.Fatal("use --nsqd-tcp-address or --lookupd-http-address not both") } if len(destNsqdTCPAddrs) == 0 { log.Fatal("--destination-nsqd-tcp-address required") } switch *mode { case "round-robin": selectedMode = ModeRoundRobin case "hostpool", "epsilon-greedy": selectedMode = ModeHostPool } termChan := make(chan os.Signal, 1) signal.Notify(termChan, syscall.SIGINT, syscall.SIGTERM) defaultUA := fmt.Sprintf("nsq_to_nsq/%s go-nsq/%s", version.Binary, nsq.VERSION) cCfg.UserAgent = defaultUA cCfg.MaxInFlight = *maxInFlight pCfg.UserAgent = defaultUA producers := make(map[string]*nsq.Producer) for _, addr := range destNsqdTCPAddrs { producer, err := nsq.NewProducer(addr, pCfg) if err != nil { log.Fatalf("failed creating producer %s", err) } producers[addr] = producer } perAddressStatus := make(map[string]*timer_metrics.TimerMetrics) if len(destNsqdTCPAddrs) == 1 { // disable since there is only one address perAddressStatus[destNsqdTCPAddrs[0]] = timer_metrics.NewTimerMetrics(0, "") } else { for _, a := range destNsqdTCPAddrs { perAddressStatus[a] = timer_metrics.NewTimerMetrics(*statusEvery, fmt.Sprintf("[%s]:", a)) } } hostPool := hostpool.New(destNsqdTCPAddrs) if *mode == "epsilon-greedy" { hostPool = hostpool.NewEpsilonGreedy(destNsqdTCPAddrs, 0, &hostpool.LinearEpsilonValueCalculator{}) } var consumerList []*nsq.Consumer publisher := &PublishHandler{ addresses: destNsqdTCPAddrs, producers: producers, mode: selectedMode, hostPool: hostPool, respChan: make(chan *nsq.ProducerTransaction, len(destNsqdTCPAddrs)), perAddressStatus: perAddressStatus, timermetrics: timer_metrics.NewTimerMetrics(*statusEvery, "[aggregate]:"), } for _, topic := range topics { consumer, err := nsq.NewConsumer(topic, *channel, cCfg) consumerList = append(consumerList, consumer) if err != nil { log.Fatal(err) } publishTopic := topic if *destTopic != "" { publishTopic = *destTopic } topicHandler := &TopicHandler{ publishHandler: publisher, destinationTopic: publishTopic, } consumer.AddConcurrentHandlers(topicHandler, len(destNsqdTCPAddrs)) } for i := 0; i < len(destNsqdTCPAddrs); i++ { go publisher.responder() } for _, consumer := range consumerList { err := consumer.ConnectToNSQDs(nsqdTCPAddrs) if err != nil { log.Fatal(err) } } for _, consumer := range consumerList { err := consumer.ConnectToNSQLookupds(lookupdHTTPAddrs) if err != nil { log.Fatal(err) } } <-termChan // wait for signal for _, consumer := range consumerList { consumer.Stop() } for _, consumer := range consumerList { <-consumer.StopChan } } ================================================ FILE: apps/nsqadmin/main.go ================================================ package main import ( "flag" "fmt" "os" "path/filepath" "sync" "syscall" "github.com/BurntSushi/toml" "github.com/judwhite/go-svc" "github.com/mreiferson/go-options" "github.com/nsqio/nsq/internal/app" "github.com/nsqio/nsq/internal/lg" "github.com/nsqio/nsq/internal/version" "github.com/nsqio/nsq/nsqadmin" ) func nsqadminFlagSet(opts *nsqadmin.Options) *flag.FlagSet { flagSet := flag.NewFlagSet("nsqadmin", flag.ExitOnError) flagSet.String("config", "", "path to config file") flagSet.Bool("version", false, "print version string") logLevel := opts.LogLevel flagSet.Var(&logLevel, "log-level", "set log verbosity: debug, info, warn, error, or fatal") flagSet.String("log-prefix", "[nsqadmin] ", "log message prefix") flagSet.Bool("verbose", false, "[deprecated] has no effect, use --log-level") flagSet.String("http-address", opts.HTTPAddress, ": to listen on for HTTP clients") flagSet.String("base-path", opts.BasePath, "URL base path") flagSet.String("dev-static-dir", opts.DevStaticDir, "(development use only)") flagSet.String("graphite-url", opts.GraphiteURL, "graphite HTTP address") flagSet.Bool("proxy-graphite", false, "proxy HTTP requests to graphite") flagSet.String("statsd-counter-format", opts.StatsdCounterFormat, "The counter stats key formatting applied by the implementation of statsd. If no formatting is desired, set this to an empty string.") flagSet.String("statsd-gauge-format", opts.StatsdGaugeFormat, "The gauge stats key formatting applied by the implementation of statsd. If no formatting is desired, set this to an empty string.") flagSet.String("statsd-prefix", opts.StatsdPrefix, "prefix used for keys sent to statsd (%s for host replacement, must match nsqd)") flagSet.Duration("statsd-interval", opts.StatsdInterval, "time interval nsqd is configured to push to statsd (must match nsqd)") flagSet.String("notification-http-endpoint", "", "HTTP endpoint (fully qualified) to which POST notifications of admin actions will be sent") flagSet.Duration("http-client-connect-timeout", opts.HTTPClientConnectTimeout, "timeout for HTTP connect") flagSet.Duration("http-client-request-timeout", opts.HTTPClientRequestTimeout, "timeout for HTTP request") flagSet.Bool("http-client-tls-insecure-skip-verify", false, "configure the HTTP client to skip verification of TLS certificates") flagSet.String("http-client-tls-root-ca-file", "", "path to CA file for the HTTP client") flagSet.String("http-client-tls-cert", "", "path to certificate file for the HTTP client") flagSet.String("http-client-tls-key", "", "path to key file for the HTTP client") flagSet.String("allow-config-from-cidr", opts.AllowConfigFromCIDR, "A CIDR from which to allow HTTP requests to the /config endpoint") flagSet.String("acl-http-header", opts.ACLHTTPHeader, "HTTP header to check for authenticated admin users") nsqlookupdHTTPAddresses := app.StringArray{} flagSet.Var(&nsqlookupdHTTPAddresses, "lookupd-http-address", "lookupd HTTP address (may be given multiple times)") nsqdHTTPAddresses := app.StringArray{} flagSet.Var(&nsqdHTTPAddresses, "nsqd-http-address", "nsqd HTTP address (may be given multiple times)") adminUsers := app.StringArray{} flagSet.Var(&adminUsers, "admin-user", "admin user (may be given multiple times; if specified, only these users will be able to perform privileged actions; acl-http-header is used to determine the authenticated user)") return flagSet } type program struct { once sync.Once nsqadmin *nsqadmin.NSQAdmin } func main() { prg := &program{} if err := svc.Run(prg, syscall.SIGINT, syscall.SIGTERM); err != nil { logFatal("%s", err) } } func (p *program) Init(env svc.Environment) error { if env.IsWindowsService() { dir := filepath.Dir(os.Args[0]) return os.Chdir(dir) } return nil } func (p *program) Start() error { opts := nsqadmin.NewOptions() flagSet := nsqadminFlagSet(opts) flagSet.Parse(os.Args[1:]) if flagSet.Lookup("version").Value.(flag.Getter).Get().(bool) { fmt.Println(version.String("nsqadmin")) os.Exit(0) } var cfg config configFile := flagSet.Lookup("config").Value.String() if configFile != "" { _, err := toml.DecodeFile(configFile, &cfg) if err != nil { logFatal("failed to load config file %s - %s", configFile, err) } } cfg.Validate() options.Resolve(opts, flagSet, cfg) nsqadmin, err := nsqadmin.New(opts) if err != nil { logFatal("failed to instantiate nsqadmin - %s", err) } p.nsqadmin = nsqadmin go func() { err := p.nsqadmin.Main() if err != nil { p.Stop() os.Exit(1) } }() return nil } func (p *program) Stop() error { p.once.Do(func() { p.nsqadmin.Exit() }) return nil } func logFatal(f string, args ...interface{}) { lg.LogFatal("[nsqadmin] ", f, args...) } ================================================ FILE: apps/nsqadmin/main_test.go ================================================ package main import ( "testing" "github.com/mreiferson/go-options" "github.com/nsqio/nsq/internal/lg" "github.com/nsqio/nsq/internal/test" "github.com/nsqio/nsq/nsqadmin" ) func TestConfigFlagParsing(t *testing.T) { opts := nsqadmin.NewOptions() opts.Logger = test.NewTestLogger(t) flagSet := nsqadminFlagSet(opts) flagSet.Parse([]string{}) cfg := config{"log_level": "debug"} cfg.Validate() options.Resolve(opts, flagSet, cfg) if opts.LogLevel != lg.DEBUG { t.Fatalf("log level: want debug, got %s", opts.LogLevel.String()) } } ================================================ FILE: apps/nsqadmin/options.go ================================================ package main import ( "fmt" "github.com/nsqio/nsq/internal/lg" ) type config map[string]interface{} // Validate settings in the config file, and fatal on errors func (cfg config) Validate() { if v, exists := cfg["log_level"]; exists { var t lg.LogLevel err := t.Set(fmt.Sprintf("%v", v)) if err == nil { cfg["log_level"] = t } else { logFatal("failed parsing log_level %+v", v) } } } ================================================ FILE: apps/nsqd/README.md ================================================ ## nsqd `nsqd` is the daemon that receives, queues, and delivers messages to clients. Read the [docs](https://nsq.io/components/nsqd.html). ================================================ FILE: apps/nsqd/main.go ================================================ package main import ( "context" "flag" "fmt" "math/rand" "os" "sync" "syscall" "time" "github.com/BurntSushi/toml" "github.com/judwhite/go-svc" "github.com/mreiferson/go-options" "github.com/nsqio/nsq/internal/lg" "github.com/nsqio/nsq/internal/version" "github.com/nsqio/nsq/nsqd" ) type program struct { once sync.Once nsqd *nsqd.NSQD } func main() { prg := &program{} if err := svc.Run(prg, syscall.SIGINT, syscall.SIGTERM); err != nil { logFatal("%s", err) } } func (p *program) Init(env svc.Environment) error { opts := nsqd.NewOptions() flagSet := nsqdFlagSet(opts) flagSet.Parse(os.Args[1:]) rand.Seed(time.Now().UTC().UnixNano()) if flagSet.Lookup("version").Value.(flag.Getter).Get().(bool) { fmt.Println(version.String("nsqd")) os.Exit(0) } var cfg config configFile := flagSet.Lookup("config").Value.String() if configFile != "" { _, err := toml.DecodeFile(configFile, &cfg) if err != nil { logFatal("failed to load config file %s - %s", configFile, err) } } cfg.Validate() options.Resolve(opts, flagSet, cfg) applyBackwardCompatibility(opts, flagSet) nsqd, err := nsqd.New(opts) if err != nil { logFatal("failed to instantiate nsqd - %s", err) } p.nsqd = nsqd return nil } func (p *program) Start() error { err := p.nsqd.LoadMetadata() if err != nil { logFatal("failed to load metadata - %s", err) } err = p.nsqd.PersistMetadata() if err != nil { logFatal("failed to persist metadata - %s", err) } go func() { err := p.nsqd.Main() if err != nil { p.Stop() os.Exit(1) } }() return nil } func (p *program) Stop() error { p.once.Do(func() { p.nsqd.Exit() }) return nil } func (p *program) Handle(s os.Signal) error { return svc.ErrStop } // Context returns a context that will be canceled when nsqd initiates the shutdown func (p *program) Context() context.Context { return p.nsqd.Context() } func logFatal(f string, args ...interface{}) { lg.LogFatal("[nsqd] ", f, args...) } // applyBackwardCompatibility applies backward compatibility rules to options after flag resolution func applyBackwardCompatibility(opts *nsqd.Options, flagSet *flag.FlagSet) { // when max-defer-timeout was not explicitly set, refer to the max-req-timeout value if flag := flagSet.Lookup("max-defer-timeout"); flag != nil && flag.Value.String() == flag.DefValue { opts.MaxDeferTimeout = opts.MaxReqTimeout } // ... other backward compatibility rules can be added here } ================================================ FILE: apps/nsqd/main_test.go ================================================ package main import ( "crypto/tls" "os" "testing" "github.com/BurntSushi/toml" "github.com/mreiferson/go-options" "github.com/nsqio/nsq/internal/lg" "github.com/nsqio/nsq/internal/test" "github.com/nsqio/nsq/nsqd" ) func TestConfigFlagParsing(t *testing.T) { opts := nsqd.NewOptions() opts.Logger = test.NewTestLogger(t) flagSet := nsqdFlagSet(opts) flagSet.Parse([]string{}) var cfg config f, err := os.Open("../../contrib/nsqd.cfg.example") if err != nil { t.Fatalf("%s", err) } defer f.Close() toml.NewDecoder(f).Decode(&cfg) cfg["log_level"] = "debug" cfg.Validate() options.Resolve(opts, flagSet, cfg) nsqd.New(opts) if opts.TLSMinVersion != tls.VersionTLS10 { t.Errorf("min %#v not expected %#v", opts.TLSMinVersion, tls.VersionTLS10) } if opts.LogLevel != lg.DEBUG { t.Fatalf("log level: want debug, got %s", opts.LogLevel.String()) } } ================================================ FILE: apps/nsqd/options.go ================================================ package main import ( "crypto/tls" "flag" "fmt" "strconv" "strings" "github.com/nsqio/nsq/internal/app" "github.com/nsqio/nsq/internal/lg" "github.com/nsqio/nsq/nsqd" ) type tlsRequiredOption int func (t *tlsRequiredOption) Set(s string) error { s = strings.ToLower(s) if s == "tcp-https" { *t = nsqd.TLSRequiredExceptHTTP return nil } required, err := strconv.ParseBool(s) if required { *t = nsqd.TLSRequired } else { *t = nsqd.TLSNotRequired } return err } func (t *tlsRequiredOption) Get() interface{} { return int(*t) } func (t *tlsRequiredOption) String() string { return strconv.FormatInt(int64(*t), 10) } func (t *tlsRequiredOption) IsBoolFlag() bool { return true } type tlsMinVersionOption uint16 var tlsVersionTable = []struct { val uint16 str string }{ {tls.VersionTLS10, "tls1.0"}, {tls.VersionTLS11, "tls1.1"}, {tls.VersionTLS12, "tls1.2"}, {tls.VersionTLS13, "tls1.3"}, } func (t *tlsMinVersionOption) Set(s string) error { s = strings.ToLower(s) if s == "" { return nil } for _, v := range tlsVersionTable { if s == v.str { *t = tlsMinVersionOption(v.val) return nil } } return fmt.Errorf("unknown tlsVersionOption %q", s) } func (t *tlsMinVersionOption) Get() interface{} { return uint16(*t) } func (t *tlsMinVersionOption) String() string { for _, v := range tlsVersionTable { if uint16(*t) == v.val { return v.str } } return strconv.FormatInt(int64(*t), 10) } type config map[string]interface{} // Validate settings in the config file, and fatal on errors func (cfg config) Validate() { // special validation/translation if v, exists := cfg["tls_required"]; exists { var t tlsRequiredOption err := t.Set(fmt.Sprintf("%v", v)) if err == nil { cfg["tls_required"] = t.String() } else { logFatal("failed parsing tls_required %+v", v) } } if v, exists := cfg["tls_min_version"]; exists { var t tlsMinVersionOption err := t.Set(fmt.Sprintf("%v", v)) if err == nil { newVal := fmt.Sprintf("%v", t.Get()) if newVal != "0" { cfg["tls_min_version"] = newVal } else { delete(cfg, "tls_min_version") } } else { logFatal("failed parsing tls_min_version %+v", v) } } if v, exists := cfg["log_level"]; exists { var t lg.LogLevel err := t.Set(fmt.Sprintf("%v", v)) if err == nil { cfg["log_level"] = t } else { logFatal("failed parsing log_level %+v", v) } } } func nsqdFlagSet(opts *nsqd.Options) *flag.FlagSet { flagSet := flag.NewFlagSet("nsqd", flag.ExitOnError) // basic options flagSet.Bool("version", false, "print version string") flagSet.String("config", "", "path to config file") logLevel := opts.LogLevel flagSet.Var(&logLevel, "log-level", "set log verbosity: debug, info, warn, error, or fatal") flagSet.String("log-prefix", "[nsqd] ", "log message prefix") flagSet.Bool("verbose", false, "[deprecated] has no effect, use --log-level") flagSet.Int64("node-id", opts.ID, "unique part for message IDs, (int) in range [0,1024) (default is hash of hostname)") flagSet.Bool("worker-id", false, "[deprecated] use --node-id") flagSet.String("https-address", opts.HTTPSAddress, ": to listen on for HTTPS clients") flagSet.String("http-address", opts.HTTPAddress, "address to listen on for HTTP clients (: for TCP/IP or for unix socket)") flagSet.String("tcp-address", opts.TCPAddress, "address to listen on for TCP clients (: for TCP/IP or for unix socket)") authHTTPAddresses := app.StringArray{} flagSet.Var(&authHTTPAddresses, "auth-http-address", ": or a full url to query auth server (may be given multiple times)") flagSet.String("auth-http-request-method", opts.AuthHTTPRequestMethod, "HTTP method to use for auth server requests") flagSet.String("broadcast-address", opts.BroadcastAddress, "address that will be registered with lookupd (defaults to the OS hostname)") flagSet.Int("broadcast-tcp-port", opts.BroadcastTCPPort, "TCP port that will be registered with lookupd (defaults to the TCP port that this nsqd is listening on)") flagSet.Int("broadcast-http-port", opts.BroadcastHTTPPort, "HTTP port that will be registered with lookupd (defaults to the HTTP port that this nsqd is listening on)") lookupdTCPAddrs := app.StringArray{} flagSet.Var(&lookupdTCPAddrs, "lookupd-tcp-address", "lookupd TCP address (may be given multiple times)") flagSet.Duration("http-client-connect-timeout", opts.HTTPClientConnectTimeout, "timeout for HTTP connect") flagSet.Duration("http-client-request-timeout", opts.HTTPClientRequestTimeout, "timeout for HTTP request") flagSet.String("topology-region", opts.TopologyRegion, "A region represents a larger domain, made up of one or more zones for preferring closer consumer") flagSet.String("topology-zone", opts.TopologyZone, "A zone represents a logical failure domain for preferring closer consumer") // diskqueue options flagSet.String("data-path", opts.DataPath, "path to store disk-backed messages") flagSet.Int64("mem-queue-size", opts.MemQueueSize, "number of messages to keep in memory (per topic/channel)") flagSet.Int64("max-bytes-per-file", opts.MaxBytesPerFile, "number of bytes per diskqueue file before rolling") flagSet.Int64("sync-every", opts.SyncEvery, "number of messages per diskqueue fsync") flagSet.Duration("sync-timeout", opts.SyncTimeout, "duration of time per diskqueue fsync") flagSet.Int("queue-scan-worker-pool-max", opts.QueueScanWorkerPoolMax, "max concurrency for checking in-flight and deferred message timeouts") flagSet.Int("queue-scan-selection-count", opts.QueueScanSelectionCount, "number of channels to check per cycle (every 100ms) for in-flight and deferred timeouts") // msg and command options flagSet.Duration("msg-timeout", opts.MsgTimeout, "default duration to wait before auto-requeing a message") flagSet.Duration("max-msg-timeout", opts.MaxMsgTimeout, "maximum duration before a message will timeout") flagSet.Int64("max-msg-size", opts.MaxMsgSize, "maximum size of a single message in bytes") flagSet.Duration("max-req-timeout", opts.MaxReqTimeout, "maximum requeuing timeout for a message") flagSet.Int64("max-body-size", opts.MaxBodySize, "maximum size of a single command body") flagSet.Duration("max-defer-timeout", opts.MaxDeferTimeout, "maximum duration when deferring a message") // client overridable configuration options flagSet.Duration("max-heartbeat-interval", opts.MaxHeartbeatInterval, "maximum client configurable duration of time between client heartbeats") flagSet.Int64("max-rdy-count", opts.MaxRdyCount, "maximum RDY count for a client") flagSet.Int64("max-output-buffer-size", opts.MaxOutputBufferSize, "maximum client configurable size (in bytes) for a client output buffer") flagSet.Duration("max-output-buffer-timeout", opts.MaxOutputBufferTimeout, "maximum client configurable duration of time between flushing to a client") flagSet.Duration("min-output-buffer-timeout", opts.MinOutputBufferTimeout, "minimum client configurable duration of time between flushing to a client") flagSet.Duration("output-buffer-timeout", opts.OutputBufferTimeout, "default duration of time between flushing data to clients") flagSet.Int("max-channel-consumers", opts.MaxChannelConsumers, "maximum channel consumer connection count per nsqd instance (default 0, i.e., unlimited)") // statsd integration options flagSet.String("statsd-address", opts.StatsdAddress, "UDP : of a statsd daemon for pushing stats") flagSet.Duration("statsd-interval", opts.StatsdInterval, "duration between pushing to statsd") flagSet.Bool("statsd-mem-stats", opts.StatsdMemStats, "toggle sending memory and GC stats to statsd") flagSet.String("statsd-prefix", opts.StatsdPrefix, "prefix used for keys sent to statsd (%s for host replacement)") flagSet.Int("statsd-udp-packet-size", opts.StatsdUDPPacketSize, "the size in bytes of statsd UDP packets") flagSet.Bool("statsd-exclude-ephemeral", opts.StatsdExcludeEphemeral, "Skip ephemeral topics and channels when sending stats to statsd") // End to end percentile flags e2eProcessingLatencyPercentiles := app.FloatArray{} flagSet.Var(&e2eProcessingLatencyPercentiles, "e2e-processing-latency-percentile", "message processing time percentiles (as float (0, 1.0]) to track (can be specified multiple times or comma separated '1.0,0.99,0.95', default none)") flagSet.Duration("e2e-processing-latency-window-time", opts.E2EProcessingLatencyWindowTime, "calculate end to end latency quantiles for this duration of time (ie: 60s would only show quantile calculations from the past 60 seconds)") // TLS config flagSet.String("tls-cert", opts.TLSCert, "path to certificate file") flagSet.String("tls-key", opts.TLSKey, "path to key file") flagSet.String("tls-client-auth-policy", opts.TLSClientAuthPolicy, "client certificate auth policy ('require' or 'require-verify')") flagSet.String("tls-root-ca-file", opts.TLSRootCAFile, "path to certificate authority file") tlsRequired := tlsRequiredOption(opts.TLSRequired) tlsMinVersion := tlsMinVersionOption(opts.TLSMinVersion) flagSet.Var(&tlsRequired, "tls-required", "require TLS for client connections (true, false, tcp-https)") flagSet.Var(&tlsMinVersion, "tls-min-version", "minimum SSL/TLS version acceptable ('ssl3.0', 'tls1.0', 'tls1.1', 'tls1.2' or 'tls1.3')") // compression flagSet.Bool("deflate", opts.DeflateEnabled, "enable deflate feature negotiation (client compression)") flagSet.Int("max-deflate-level", opts.MaxDeflateLevel, "max deflate compression level a client can negotiate (> values == > nsqd CPU usage)") flagSet.Bool("snappy", opts.SnappyEnabled, "enable snappy feature negotiation (client compression)") experiments := app.StringArray{} var validExperiments []string for _, e := range nsqd.AllExperiments { validExperiments = append(validExperiments, fmt.Sprintf("%q", string(e))) } flagSet.Var(&experiments, "enable-experiment", fmt.Sprintf("enable experimental feature (may be given multiple times) (valid options: %s)", strings.Join(validExperiments, ", "))) return flagSet } ================================================ FILE: apps/nsqlookupd/README.md ================================================ ## nsqlookupd `nsqlookupd` is the daemon that manages topology metadata and serves client requests to discover the location of topics at runtime. Read the [docs](https://nsq.io/components/nsqlookupd.html). ================================================ FILE: apps/nsqlookupd/main.go ================================================ package main import ( "flag" "fmt" "os" "path/filepath" "sync" "syscall" "github.com/BurntSushi/toml" "github.com/judwhite/go-svc" "github.com/mreiferson/go-options" "github.com/nsqio/nsq/internal/lg" "github.com/nsqio/nsq/internal/version" "github.com/nsqio/nsq/nsqlookupd" ) func nsqlookupdFlagSet(opts *nsqlookupd.Options) *flag.FlagSet { flagSet := flag.NewFlagSet("nsqlookupd", flag.ExitOnError) flagSet.String("config", "", "path to config file") flagSet.Bool("version", false, "print version string") logLevel := opts.LogLevel flagSet.Var(&logLevel, "log-level", "set log verbosity: debug, info, warn, error, or fatal") flagSet.String("log-prefix", "[nsqlookupd] ", "log message prefix") flagSet.Bool("verbose", false, "[deprecated] has no effect, use --log-level") flagSet.String("tcp-address", opts.TCPAddress, ": to listen on for TCP clients") flagSet.String("http-address", opts.HTTPAddress, ": to listen on for HTTP clients") flagSet.String("broadcast-address", opts.BroadcastAddress, "address of this lookupd node, (default to the OS hostname)") flagSet.Duration("inactive-producer-timeout", opts.InactiveProducerTimeout, "duration of time a producer will remain in the active list since its last ping") flagSet.Duration("tombstone-lifetime", opts.TombstoneLifetime, "duration of time a producer will remain tombstoned if registration remains") return flagSet } type program struct { once sync.Once nsqlookupd *nsqlookupd.NSQLookupd } func main() { prg := &program{} if err := svc.Run(prg, syscall.SIGINT, syscall.SIGTERM); err != nil { logFatal("%s", err) } } func (p *program) Init(env svc.Environment) error { if env.IsWindowsService() { dir := filepath.Dir(os.Args[0]) return os.Chdir(dir) } return nil } func (p *program) Start() error { opts := nsqlookupd.NewOptions() flagSet := nsqlookupdFlagSet(opts) flagSet.Parse(os.Args[1:]) if flagSet.Lookup("version").Value.(flag.Getter).Get().(bool) { fmt.Println(version.String("nsqlookupd")) os.Exit(0) } var cfg config configFile := flagSet.Lookup("config").Value.String() if configFile != "" { _, err := toml.DecodeFile(configFile, &cfg) if err != nil { logFatal("failed to load config file %s - %s", configFile, err) } } cfg.Validate() options.Resolve(opts, flagSet, cfg) nsqlookupd, err := nsqlookupd.New(opts) if err != nil { logFatal("failed to instantiate nsqlookupd", err) } p.nsqlookupd = nsqlookupd go func() { err := p.nsqlookupd.Main() if err != nil { p.Stop() os.Exit(1) } }() return nil } func (p *program) Stop() error { p.once.Do(func() { p.nsqlookupd.Exit() }) return nil } func logFatal(f string, args ...interface{}) { lg.LogFatal("[nsqlookupd] ", f, args...) } ================================================ FILE: apps/nsqlookupd/main_test.go ================================================ package main import ( "testing" "github.com/mreiferson/go-options" "github.com/nsqio/nsq/internal/lg" "github.com/nsqio/nsq/internal/test" "github.com/nsqio/nsq/nsqlookupd" ) func TestConfigFlagParsing(t *testing.T) { opts := nsqlookupd.NewOptions() opts.Logger = test.NewTestLogger(t) flagSet := nsqlookupdFlagSet(opts) flagSet.Parse([]string{}) cfg := config{"log_level": "debug"} cfg.Validate() options.Resolve(opts, flagSet, cfg) if opts.LogLevel != lg.DEBUG { t.Fatalf("log level: want debug, got %s", opts.LogLevel.String()) } } ================================================ FILE: apps/nsqlookupd/options.go ================================================ package main import ( "fmt" "github.com/nsqio/nsq/internal/lg" ) type config map[string]interface{} // Validate settings in the config file, and fatal on errors func (cfg config) Validate() { if v, exists := cfg["log_level"]; exists { var t lg.LogLevel err := t.Set(fmt.Sprintf("%v", v)) if err == nil { cfg["log_level"] = t } else { logFatal("failed parsing log_level %+v", v) } } } ================================================ FILE: apps/to_nsq/README.md ================================================ # to_nsq A tool for publishing to an nsq topic with data from `stdin`. ## Usage ``` Usage of ./to_nsq: -delimiter string character to split input from stdin (default "\n") -nsqd-tcp-address value destination nsqd TCP address (may be given multiple times) -producer-opt value option to passthrough to nsq.Producer (may be given multiple times, http://godoc.org/github.com/nsqio/go-nsq#Config) -rate int Throttle messages to n/second. 0 to disable -topic string NSQ topic to publish to ``` ### Examples Publish each line of a file: ```bash $ cat source.txt | to_nsq -topic="topic" -nsqd-tcp-address="127.0.0.1:4150" ``` Publish three messages, in one go: ```bash $ echo "one,two,three" | to_nsq -delimiter="," -topic="topic" -nsqd-tcp-address="127.0.0.1:4150" ``` ================================================ FILE: apps/to_nsq/to_nsq.go ================================================ // This is an NSQ client that publishes incoming messages from // stdin to the specified topic. package main import ( "bufio" "flag" "fmt" "io" "log" "os" "os/signal" "sync/atomic" "syscall" "time" "github.com/nsqio/go-nsq" "github.com/nsqio/nsq/internal/app" "github.com/nsqio/nsq/internal/version" ) var ( topic = flag.String("topic", "", "NSQ topic to publish to") delimiter = flag.String("delimiter", "\n", "character to split input from stdin") destNsqdTCPAddrs = app.StringArray{} ) func init() { flag.Var(&destNsqdTCPAddrs, "nsqd-tcp-address", "destination nsqd TCP address (may be given multiple times)") } func main() { cfg := nsq.NewConfig() flag.Var(&nsq.ConfigFlag{cfg}, "producer-opt", "option to passthrough to nsq.Producer (may be given multiple times, http://godoc.org/github.com/nsqio/go-nsq#Config)") rate := flag.Int64("rate", 0, "Throttle messages to n/second. 0 to disable") flag.Parse() if len(*topic) == 0 { log.Fatal("--topic required") } if len(*delimiter) != 1 { log.Fatal("--delimiter must be a single byte") } stopChan := make(chan bool) termChan := make(chan os.Signal, 1) signal.Notify(termChan, syscall.SIGINT, syscall.SIGTERM) cfg.UserAgent = fmt.Sprintf("to_nsq/%s go-nsq/%s", version.Binary, nsq.VERSION) // make the producers producers := make(map[string]*nsq.Producer) for _, addr := range destNsqdTCPAddrs { producer, err := nsq.NewProducer(addr, cfg) if err != nil { log.Fatalf("failed to create nsq.Producer - %s", err) } producers[addr] = producer } if len(producers) == 0 { log.Fatal("--nsqd-tcp-address required") } throttleEnabled := *rate >= 1 balance := int64(1) // avoid divide by 0 if !throttleEnabled var interval time.Duration if throttleEnabled { interval = time.Second / time.Duration(*rate) } go func() { if !throttleEnabled { return } log.Printf("Throttling messages rate to max:%d/second", *rate) // every tick increase the number of messages we can send for range time.Tick(interval) { n := atomic.AddInt64(&balance, 1) // if we build up more than 1s of capacity just bound to that if n > int64(*rate) { atomic.StoreInt64(&balance, int64(*rate)) } } }() r := bufio.NewReader(os.Stdin) delim := (*delimiter)[0] go func() { for { var err error if throttleEnabled { currentBalance := atomic.LoadInt64(&balance) if currentBalance <= 0 { time.Sleep(interval) } err = readAndPublish(r, delim, producers) atomic.AddInt64(&balance, -1) } else { err = readAndPublish(r, delim, producers) } if err != nil { if err != io.EOF { log.Fatal(err) } close(stopChan) break } } }() select { case <-termChan: case <-stopChan: } for _, producer := range producers { producer.Stop() } } // readAndPublish reads to the delim from r and publishes the bytes // to the map of producers. func readAndPublish(r *bufio.Reader, delim byte, producers map[string]*nsq.Producer) error { line, readErr := r.ReadBytes(delim) if len(line) > 0 { // trim the delimiter line = line[:len(line)-1] } if len(line) == 0 { return readErr } for _, producer := range producers { err := producer.Publish(*topic, line) if err != nil { return err } } return readErr } ================================================ FILE: bench/bench.py ================================================ #!/usr/bin/env python3 # # This script bootstraps an NSQ cluster in EC2 and runs benchmarks. # # Requires python3 and the following packages: # - boto3 # - paramiko # - tornado # # AWS authentication is delegated entirely to the boto3 environment, see: # # https://boto3.amazonaws.com/v1/documentation/api/latest/guide/configuration.html # # EC2 instances are launched into EC2 Classic, expecting a 'default' security group # that allows allows SSH (port 22) from 0.0.0.0/0 and an EC2 key pair created # (named 'default', but configurable via --ssh-key-name). # import sys import logging import time import datetime import socket import warnings import hashlib import boto3 import paramiko.client import paramiko.ssh_exception import tornado.options def ssh_connect_with_retries(host, retries=3, timeout=30): for i in range(retries): try: ssh_client = paramiko.client.SSHClient() ssh_client.set_missing_host_key_policy(paramiko.client.WarningPolicy()) ssh_client.connect(host, username='ubuntu', timeout=timeout) return ssh_client except (socket.error, paramiko.ssh_exception.SSHException): if i == retries - 1: raise logging.warning('... re-trying to connect to %s:%d in 15s', host, 22) time.sleep(15) def ssh_cmd_async(ssh_client, cmd): transport = ssh_client.get_transport() chan = transport.open_session() chan.exec_command(cmd) return chan def ssh_cmd(ssh_client, cmd, timeout=2): transport = ssh_client.get_transport() chan = transport.open_session() chan.settimeout(timeout) chan.exec_command(cmd) stdout = b'' stderr = b'' while True: if chan.recv_ready(): stdout += chan.recv(4096) continue if chan.recv_stderr_ready(): stderr += chan.recv_stderr(4096) continue if chan.exit_status_ready(): exit_status = chan.recv_exit_status() break time.sleep(0.1) if exit_status != 0: raise Exception('%r' % stderr) return stdout, stderr def get_session(): return boto3.session.Session(region_name=tornado.options.options.region) def _bootstrap(addr): commit = tornado.options.options.commit golang_version = tornado.options.options.golang_version ssh_client = ssh_connect_with_retries(addr) for cmd in [ 'wget https://storage.googleapis.com/golang/go%s.linux-amd64.tar.gz' % golang_version, 'sudo -S tar -C /usr/local -xzf go%s.linux-amd64.tar.gz' % golang_version, 'sudo -S apt-get update', 'sudo -S apt-get -y install git mercurial', 'mkdir -p go/src/github.com/nsqio', 'cd go/src/github.com/nsqio && git clone https://github.com/nsqio/nsq', 'cd go/src/github.com/nsqio/nsq && git checkout %s' % commit, 'cd go/src/github.com/nsqio/nsq/apps/nsqd && GO111MODULE=on /usr/local/go/bin/go build', 'cd go/src/github.com/nsqio/nsq/bench/bench_writer && GO111MODULE=on /usr/local/go/bin/go build', 'cd go/src/github.com/nsqio/nsq/bench/bench_reader && GO111MODULE=on /usr/local/go/bin/go build', 'sudo -S mkdir -p /mnt/nsq', 'sudo -S chmod 777 /mnt/nsq']: ssh_cmd(ssh_client, cmd, timeout=10) def bootstrap(): session = get_session() ec2 = session.resource('ec2') total_count = tornado.options.options.nsqd_count + tornado.options.options.worker_count logging.info('launching %d instances', total_count) instances = ec2.create_instances( ImageId=tornado.options.options.ami, MinCount=total_count, MaxCount=total_count, KeyName=tornado.options.options.ssh_key_name, InstanceType=tornado.options.options.instance_type, SecurityGroups=['default']) logging.info('waiting for instances to launch...') while any(i.state['Name'] != 'running' for i in instances): waiting_for = [i.id for i in instances if i.state['Name'] != 'running'] logging.info('... sleeping for 5s (waiting for %s)', ', '.join(waiting_for)) time.sleep(5) for instance in instances: instance.load() for instance in instances: if not instance.tags: instance.create_tags(Tags=[{'Key': 'nsq_bench', 'Value': '1'}]) try: c = 0 for i in instances: c += 1 logging.info('(%d) bootstrapping %s (%s)', c, i.public_dns_name, i.id) _bootstrap(i.public_dns_name) except Exception: logging.exception('bootstrap failed') decomm() def run(): instances = _find_instances() logging.info('launching nsqd on %d host(s)', tornado.options.options.nsqd_count) nsqd_chans = [] nsqd_hosts = instances[:tornado.options.options.nsqd_count] for instance in nsqd_hosts: try: ssh_client = ssh_connect_with_retries(instance.public_dns_name) for cmd in [ 'sudo -S pkill -f nsqd', 'sudo -S rm -f /mnt/nsq/*.dat', 'GOMAXPROCS=32 ./go/src/github.com/nsqio/nsq/apps/nsqd/nsqd \ --data-path=/mnt/nsq \ --mem-queue-size=10000000 \ --max-rdy-count=%s' % (tornado.options.options.rdy)]: nsqd_chans.append((ssh_client, ssh_cmd_async(ssh_client, cmd))) except Exception: logging.exception('failed') nsqd_tcp_addrs = [i.public_dns_name for i in nsqd_hosts] dt = datetime.datetime.utcnow() deadline = dt + datetime.timedelta(seconds=30) logging.info('launching %d producer(s) on %d host(s)', tornado.options.options.nsqd_count * tornado.options.options.worker_count, tornado.options.options.worker_count) worker_chans = [] producer_hosts = instances[tornado.options.options.nsqd_count:] for instance in producer_hosts: for nsqd_tcp_addr in nsqd_tcp_addrs: topic = hashlib.md5(instance.public_dns_name.encode('utf-8')).hexdigest() try: ssh_client = ssh_connect_with_retries(instance.public_dns_name) for cmd in [ 'GOMAXPROCS=2 \ ./go/src/github.com/nsqio/nsq/bench/bench_writer/bench_writer \ --topic=%s --nsqd-tcp-address=%s:4150 --deadline=\'%s\' --size=%d' % ( topic, nsqd_tcp_addr, deadline.strftime('%Y-%m-%d %H:%M:%S'), tornado.options.options.msg_size)]: worker_chans.append((ssh_client, ssh_cmd_async(ssh_client, cmd))) except Exception: logging.exception('failed') if tornado.options.options.mode == 'pubsub': logging.info('launching %d consumer(s) on %d host(s)', tornado.options.options.nsqd_count * tornado.options.options.worker_count, tornado.options.options.worker_count) consumer_hosts = instances[tornado.options.options.nsqd_count:] for instance in consumer_hosts: for nsqd_tcp_addr in nsqd_tcp_addrs: topic = hashlib.md5(instance.public_dns_name.encode('utf-8')).hexdigest() try: ssh_client = ssh_connect_with_retries(instance.public_dns_name) for cmd in [ 'GOMAXPROCS=8 \ ./go/src/github.com/nsqio/nsq/bench/bench_reader/bench_reader \ --topic=%s --nsqd-tcp-address=%s:4150 --deadline=\'%s\' --size=%d \ --rdy=%d' % ( topic, nsqd_tcp_addr, deadline.strftime('%Y-%m-%d %H:%M:%S'), tornado.options.options.msg_size, tornado.options.options.rdy)]: worker_chans.append((ssh_client, ssh_cmd_async(ssh_client, cmd))) except Exception: logging.exception('failed') stats = { 'bench_reader': { 'durations': [], 'mbytes': [], 'ops': [] }, 'bench_writer': { 'durations': [], 'mbytes': [], 'ops': [] } } while worker_chans: for ssh_client, chan in worker_chans[:]: if chan.recv_ready(): sys.stdout.write(chan.recv(4096)) sys.stdout.flush() continue if chan.recv_stderr_ready(): line = chan.recv_stderr(4096).decode('utf-8') if 'duration:' in line: kind = line.split(' ')[0][1:-1] parts = line.rsplit('duration:')[1].split('-') stats[kind]['durations'].append(float(parts[0].strip()[:-1])) stats[kind]['mbytes'].append(float(parts[1].strip()[:-4])) stats[kind]['ops'].append(float(parts[2].strip()[:-5])) sys.stdout.write(line) sys.stdout.flush() continue if chan.exit_status_ready(): worker_chans.remove((ssh_client, chan)) time.sleep(0.1) for kind, data in stats.items(): if not data['durations']: continue max_duration = max(data['durations']) total_mb = sum(data['mbytes']) total_ops = sum(data['ops']) logging.info('[%s] %fs - %fmb/s - %fops/s - %fus/op', kind, max_duration, total_mb, total_ops, max_duration / total_ops * 1000 * 1000) for ssh_client, chan in nsqd_chans: chan.close() def _find_instances(): session = get_session() ec2 = session.resource('ec2') return [i for i in ec2.instances.all() if i.state['Name'] == 'running' and any(t['Key'] == 'nsq_bench' for t in i.tags)] def decomm(): instances = _find_instances() logging.info('terminating instances %s' % ','.join(i.id for i in instances)) for instance in instances: instance.terminate() if __name__ == '__main__': tornado.options.define('region', type=str, default='us-east-1', help='EC2 region to launch instances') tornado.options.define('nsqd_count', type=int, default=3, help='how many nsqd instances to launch') tornado.options.define('worker_count', type=int, default=3, help='how many worker instances to launch') # ubuntu 18.04 HVM instance store us-east-1 tornado.options.define('ami', type=str, default='ami-0938f2289b3fa3f5b', help='AMI ID') tornado.options.define('ssh_key_name', type=str, default='default', help='SSH key name') tornado.options.define('instance_type', type=str, default='c3.2xlarge', help='EC2 instance type') tornado.options.define('msg_size', type=int, default=200, help='size of message') tornado.options.define('rdy', type=int, default=10000, help='RDY count to use for bench_reader') tornado.options.define('mode', type=str, default='pubsub', help='the benchmark mode (pub, pubsub)') tornado.options.define('commit', type=str, default='master', help='the git commit') tornado.options.define('golang_version', type=str, default='1.14.3', help='the go version') tornado.options.parse_command_line() logging.getLogger('paramiko').setLevel(logging.WARNING) warnings.simplefilter('ignore') cmd_name = sys.argv[-1] cmd_map = { 'bootstrap': bootstrap, 'run': run, 'decomm': decomm } cmd = cmd_map.get(cmd_name, bootstrap) sys.exit(cmd()) ================================================ FILE: bench/bench_channels/bench_channels.go ================================================ package main import ( "bufio" "flag" "fmt" "net" "sync" "time" "github.com/nsqio/go-nsq" ) var ( num = flag.Int("num", 10000, "num channels") tcpAddress = flag.String("nsqd-tcp-address", "127.0.0.1:4150", ": to connect to nsqd") ) func main() { flag.Parse() var wg sync.WaitGroup goChan := make(chan int) rdyChan := make(chan int) for j := 0; j < *num; j++ { wg.Add(1) go func(id int) { subWorker(*num, *tcpAddress, fmt.Sprintf("t%d", j), "ch", rdyChan, goChan, id) wg.Done() }(j) <-rdyChan time.Sleep(5 * time.Millisecond) } close(goChan) wg.Wait() } func subWorker(n int, tcpAddr string, topic string, channel string, rdyChan chan int, goChan chan int, id int) { conn, err := net.DialTimeout("tcp", tcpAddr, time.Second) if err != nil { panic(err.Error()) } conn.Write(nsq.MagicV2) rw := bufio.NewReadWriter(bufio.NewReader(conn), bufio.NewWriter(conn)) ci := make(map[string]interface{}) ci["client_id"] = "test" cmd, _ := nsq.Identify(ci) cmd.WriteTo(rw) nsq.Subscribe(topic, channel).WriteTo(rw) rdyCount := 1 rdy := rdyCount rdyChan <- 1 <-goChan nsq.Ready(rdyCount).WriteTo(rw) rw.Flush() nsq.ReadResponse(rw) nsq.ReadResponse(rw) for { resp, err := nsq.ReadResponse(rw) if err != nil { panic(err.Error()) } frameType, data, err := nsq.UnpackResponse(resp) if err != nil { panic(err.Error()) } if frameType == nsq.FrameTypeError { panic(string(data)) } else if frameType == nsq.FrameTypeResponse { nsq.Nop().WriteTo(rw) rw.Flush() continue } msg, err := nsq.DecodeMessage(data) if err != nil { panic(err.Error()) } nsq.Finish(msg.ID).WriteTo(rw) rdy-- if rdy == 0 { nsq.Ready(rdyCount).WriteTo(rw) rdy = rdyCount rw.Flush() } } } ================================================ FILE: bench/bench_reader/bench_reader.go ================================================ package main import ( "bufio" "errors" "flag" "fmt" "log" "net" "runtime" "sync" "sync/atomic" "time" "github.com/nsqio/go-nsq" ) var ( runfor = flag.Duration("runfor", 10*time.Second, "duration of time to run") tcpAddress = flag.String("nsqd-tcp-address", "127.0.0.1:4150", ": to connect to nsqd") size = flag.Int("size", 200, "size of messages") topic = flag.String("topic", "sub_bench", "topic to receive messages on") channel = flag.String("channel", "ch", "channel to receive messages on") deadline = flag.String("deadline", "", "deadline to start the benchmark run") rdy = flag.Int("rdy", 2500, "RDY count to use") ) var totalMsgCount int64 func main() { flag.Parse() var wg sync.WaitGroup log.SetPrefix("[bench_reader] ") goChan := make(chan int) rdyChan := make(chan int) workers := runtime.GOMAXPROCS(0) for j := 0; j < workers; j++ { wg.Add(1) go func(id int) { subWorker(*runfor, workers, *tcpAddress, *topic, *channel, rdyChan, goChan, id) wg.Done() }(j) <-rdyChan } if *deadline != "" { t, err := time.Parse("2006-01-02 15:04:05", *deadline) if err != nil { log.Fatal(err) } d := time.Until(t) log.Printf("sleeping until %s (%s)", t, d) time.Sleep(d) } start := time.Now() close(goChan) wg.Wait() end := time.Now() duration := end.Sub(start) tmc := atomic.LoadInt64(&totalMsgCount) log.Printf("duration: %s - %.03fmb/s - %.03fops/s - %.03fus/op", duration, float64(tmc*int64(*size))/duration.Seconds()/1024/1024, float64(tmc)/duration.Seconds(), float64(duration/time.Microsecond)/float64(tmc)) } func subWorker(td time.Duration, workers int, tcpAddr string, topic string, channel string, rdyChan chan int, goChan chan int, id int) { conn, err := net.DialTimeout("tcp", tcpAddr, time.Second) if err != nil { panic(err.Error()) } conn.Write(nsq.MagicV2) rw := bufio.NewReadWriter(bufio.NewReader(conn), bufio.NewWriter(conn)) ci := make(map[string]interface{}) ci["client_id"] = "reader" ci["hostname"] = "reader" ci["user_agent"] = fmt.Sprintf("bench_reader/%s", nsq.VERSION) cmd, _ := nsq.Identify(ci) cmd.WriteTo(rw) nsq.Subscribe(topic, channel).WriteTo(rw) rdyChan <- 1 <-goChan nsq.Ready(*rdy).WriteTo(rw) rw.Flush() nsq.ReadResponse(rw) nsq.ReadResponse(rw) var msgCount int64 go func() { time.Sleep(td) conn.Close() }() for { resp, err := nsq.ReadResponse(rw) if err != nil { if errors.Is(err, net.ErrClosed) { break } panic(err.Error()) } frameType, data, err := nsq.UnpackResponse(resp) if err != nil { panic(err.Error()) } if frameType == nsq.FrameTypeError { panic(string(data)) } else if frameType == nsq.FrameTypeResponse { continue } msg, err := nsq.DecodeMessage(data) if err != nil { panic(err.Error()) } nsq.Finish(msg.ID).WriteTo(rw) msgCount++ if float64(msgCount%int64(*rdy)) > float64(*rdy)*0.75 { rw.Flush() } } atomic.AddInt64(&totalMsgCount, msgCount) } ================================================ FILE: bench/bench_writer/bench_writer.go ================================================ package main import ( "bufio" "flag" "fmt" "log" "net" "runtime" "sync" "sync/atomic" "time" "github.com/nsqio/go-nsq" ) var ( runfor = flag.Duration("runfor", 10*time.Second, "duration of time to run") tcpAddress = flag.String("nsqd-tcp-address", "127.0.0.1:4150", ": to connect to nsqd") topic = flag.String("topic", "sub_bench", "topic to receive messages on") size = flag.Int("size", 200, "size of messages") batchSize = flag.Int("batch-size", 200, "batch size of messages") deadline = flag.String("deadline", "", "deadline to start the benchmark run") ) var totalMsgCount int64 func main() { flag.Parse() var wg sync.WaitGroup log.SetPrefix("[bench_writer] ") msg := make([]byte, *size) batch := make([][]byte, *batchSize) for i := range batch { batch[i] = msg } goChan := make(chan int) rdyChan := make(chan int) for j := 0; j < runtime.GOMAXPROCS(0); j++ { wg.Add(1) go func() { pubWorker(*runfor, *tcpAddress, *batchSize, batch, *topic, rdyChan, goChan) wg.Done() }() <-rdyChan } if *deadline != "" { t, err := time.Parse("2006-01-02 15:04:05", *deadline) if err != nil { log.Fatal(err) } d := time.Until(t) log.Printf("sleeping until %s (%s)", t, d) time.Sleep(d) } start := time.Now() close(goChan) wg.Wait() end := time.Now() duration := end.Sub(start) tmc := atomic.LoadInt64(&totalMsgCount) log.Printf("duration: %s - %.03fmb/s - %.03fops/s - %.03fus/op", duration, float64(tmc*int64(*size))/duration.Seconds()/1024/1024, float64(tmc)/duration.Seconds(), float64(duration/time.Microsecond)/float64(tmc)) } func pubWorker(td time.Duration, tcpAddr string, batchSize int, batch [][]byte, topic string, rdyChan chan int, goChan chan int) { conn, err := net.DialTimeout("tcp", tcpAddr, time.Second) if err != nil { panic(err.Error()) } conn.Write(nsq.MagicV2) rw := bufio.NewReadWriter(bufio.NewReader(conn), bufio.NewWriter(conn)) ci := make(map[string]interface{}) ci["client_id"] = "writer" ci["hostname"] = "writer" ci["user_agent"] = fmt.Sprintf("bench_writer/%s", nsq.VERSION) cmd, _ := nsq.Identify(ci) cmd.WriteTo(rw) rdyChan <- 1 <-goChan rw.Flush() nsq.ReadResponse(rw) var msgCount int64 endTime := time.Now().Add(td) for { cmd, _ := nsq.MultiPublish(topic, batch) _, err := cmd.WriteTo(rw) if err != nil { panic(err.Error()) } err = rw.Flush() if err != nil { panic(err.Error()) } resp, err := nsq.ReadResponse(rw) if err != nil { panic(err.Error()) } frameType, data, err := nsq.UnpackResponse(resp) if err != nil { panic(err.Error()) } if frameType == nsq.FrameTypeError { panic(string(data)) } msgCount += int64(len(batch)) if time.Now().After(endTime) { break } } atomic.AddInt64(&totalMsgCount, msgCount) } ================================================ FILE: bench/requirements.txt ================================================ tornado==4.3 paramiko==1.16.0 boto==2.39.0 ================================================ FILE: bench.sh ================================================ #!/bin/bash readonly messageSize="${1:-200}" readonly batchSize="${2:-200}" readonly memQueueSize="${3:-1000000}" readonly dataPath="${4:-}" set -e set -u echo "# using --mem-queue-size=$memQueueSize --data-path=$dataPath --size=$messageSize --batch-size=$batchSize" echo "# compiling/running nsqd" pushd apps/nsqd >/dev/null go build rm -f *.dat ./nsqd --mem-queue-size=$memQueueSize --data-path=$dataPath >/dev/null 2>&1 & nsqd_pid=$! popd >/dev/null cleanup() { kill -9 $nsqd_pid rm -f nsqd/*.dat } trap cleanup INT TERM EXIT sleep 0.3 echo "# creating topic/channel" curl --silent 'http://127.0.0.1:4151/create_topic?topic=sub_bench' >/dev/null 2>&1 curl --silent 'http://127.0.0.1:4151/create_channel?topic=sub_bench&channel=ch' >/dev/null 2>&1 echo "# compiling bench_reader/bench_writer" pushd bench >/dev/null for app in bench_reader bench_writer; do pushd $app >/dev/null go build popd >/dev/null done popd >/dev/null echo -n "PUB: " bench/bench_writer/bench_writer --size=$messageSize --batch-size=$batchSize 2>&1 curl -s -o cpu.pprof http://127.0.0.1:4151/debug/pprof/profile & pprof_pid=$! echo -n "SUB: " bench/bench_reader/bench_reader --size=$messageSize --channel=ch 2>&1 echo "waiting for pprof..." wait $pprof_pid ================================================ FILE: contrib/nsq.spec ================================================ %define name nsq %define version 1.1.1-alpha %define release 1 %define path usr/local %define group Database/Applications %define __os_install_post %{nil} Summary: nsq Name: %{name} Version: %{version} Release: %{release} Group: %{group} Packager: Matt Reiferson License: Apache BuildRoot: %{_tmppath}/%{name}-%{version}-%{release} AutoReqProv: no # we just assume you have go installed. You may or may not have an RPM to depend on. # BuildRequires: go %description NSQ - A realtime distributed messaging platform https://github.com/nsqio/nsq %prep mkdir -p $RPM_BUILD_DIR/%{name}-%{version}-%{release} cd $RPM_BUILD_DIR/%{name}-%{version}-%{release} git clone git@github.com:nsqio/nsq.git %build cd $RPM_BUILD_DIR/%{name}-%{version}-%{release}/nsq make PREFIX=/%{path} %install export DONT_STRIP=1 rm -rf $RPM_BUILD_ROOT cd $RPM_BUILD_DIR/%{name}-%{version}-%{release}/nsq make PREFIX=/${path} DESTDIR=$RPM_BUILD_ROOT install %files /%{path}/bin/nsqadmin /%{path}/bin/nsqd /%{path}/bin/nsqlookupd /%{path}/bin/nsq_to_file /%{path}/bin/nsq_to_http /%{path}/bin/nsq_to_nsq /%{path}/bin/nsq_tail /%{path}/bin/nsq_stat /%{path}/bin/to_nsq ================================================ FILE: contrib/nsqadmin.cfg.example ================================================ ## log verbosity level: debug, info, warn, error, or fatal log_level = "info" ## log message prefix (default "[nsqadmin] ") # log_prefix = "" ## HTTP header to check for authenticated admin users (default "X_Forwarded_User") # acl_http_header = "" ## admin user (may be given multiple times; if specified, only these users will be able to perform privileged actions) # admin_users = [ # "admin" # ] ## A CIDR from which to allow HTTP requests to the /config endpoint (default "127.0.0.1/8") # allow_config_from_cidr = "" ## URL base path (default "/") # base_path = "" ## timeout for HTTP connect (default 2s) # http_client_connect_timeout = "2s" ## timeout for HTTP request (default 5s) # http_client_request_timeout = "5s" ## path to certificate file for the HTTP client # http_client_tls_cert = "" ## configure the HTTP client to skip verification of TLS certificates # http_client_tls_insecure_skip_verify = false ## path to key file for the HTTP client # http_client_tls_key = "" ## path to CA file for the HTTP client # http_client_tls_root_ca_file = "" ## : to listen on for HTTP clients http_address = "0.0.0.0:4171" ## graphite HTTP address graphite_url = "" ## proxy HTTP requests to graphite proxy_graphite = false ## prefix used for keys sent to statsd (%s for host replacement, must match nsqd) statsd_prefix = "nsq.%s" ## format of statsd counter stats statsd_counter_format = "stats.counters.%s.count" ## format of statsd gauge stats statsd_gauge_format = "stats.gauges.%s" ## time interval nsqd is configured to push to statsd (must match nsqd) statsd_interval = "60s" ## HTTP endpoint (fully qualified) to which POST notifications of admin actions will be sent notification_http_endpoint = "" ## nsqlookupd HTTP addresses nsqlookupd_http_addresses = [ "127.0.0.1:4161" ] ## nsqd HTTP addresses (optional) nsqd_http_addresses = [ "127.0.0.1:4151" ] ================================================ FILE: contrib/nsqd.cfg.example ================================================ ## log verbosity level: debug, info, warn, error, or fatal log_level = "info" ## unique identifier (int) for this worker (will default to a hash of hostname) # id = 5150 ## : to listen on for TCP clients tcp_address = "0.0.0.0:4150" ## : to listen on for HTTP clients http_address = "0.0.0.0:4151" ## : to listen on for HTTPS clients # https_address = "0.0.0.0:4152" ## address that will be registered with lookupd (defaults to the OS hostname) # broadcast_address = "" ## cluster of nsqlookupd TCP addresses nsqlookupd_tcp_addresses = [ "127.0.0.1:4160" ] ## duration to wait before HTTP client connection timeout http_client_connect_timeout = "2s" ## duration to wait before HTTP client request timeout http_client_request_timeout = "5s" ## path to store disk-backed messages # data_path = "/var/lib/nsq" ## number of messages to keep in memory (per topic/channel) mem_queue_size = 10000 ## number of bytes per diskqueue file before rolling max_bytes_per_file = 104857600 ## number of messages per diskqueue fsync sync_every = 2500 ## duration of time per diskqueue fsync (time.Duration) sync_timeout = "2s" ## duration to wait before auto-requeing a message msg_timeout = "60s" ## maximum duration before a message will timeout max_msg_timeout = "15m" ## maximum size of a single message in bytes max_msg_size = 1024768 ## maximum requeuing timeout for a message max_req_timeout = "1h" ## maximum size of a single command body max_body_size = 5123840 ## maximum client configurable duration of time between client heartbeats max_heartbeat_interval = "60s" ## maximum RDY count for a client max_rdy_count = 2500 ## maximum client configurable size (in bytes) for a client output buffer max_output_buffer_size = 65536 ## maximum client configurable duration of time between flushing to a client (time.Duration) max_output_buffer_timeout = "1s" ## UDP : of a statsd daemon for pushing stats # statsd_address = "127.0.0.1:8125" ## prefix used for keys sent to statsd (%s for host replacement) statsd_prefix = "nsq.%s" ## duration between pushing to statsd (time.Duration) statsd_interval = "60s" ## toggle sending memory and GC stats to statsd statsd_mem_stats = true ## the size in bytes of statsd UDP packets # statsd_udp_packet_size = 508 ## message processing time percentiles to keep track of (float) e2e_processing_latency_percentiles = [ 1.0, 0.99, 0.95 ] ## calculate end to end latency quantiles for this duration of time (time.Duration) e2e_processing_latency_window_time = "10m" ## path to certificate file tls_cert = "" ## path to private key file tls_key = "" ## set policy on client certificate (require - client must provide certificate, ## require-verify - client must provide verifiable signed certificate) # tls_client_auth_policy = "require-verify" ## set custom root Certificate Authority # tls_root_ca_file = "" ## require client TLS upgrades tls_required = false ## minimum TLS version ("ssl3.0", "tls1.0," "tls1.1", "tls1.2") tls_min_version = "" ## enable deflate feature negotiation (client compression) deflate = true ## max deflate compression level a client can negotiate (> values == > nsqd CPU usage) max_deflate_level = 6 ## enable snappy feature negotiation (client compression) snappy = true ================================================ FILE: contrib/nsqlookupd.cfg.example ================================================ ## log verbosity level: debug, info, warn, error, or fatal log_level = "info" ## : to listen on for TCP clients tcp_address = "0.0.0.0:4160" ## : to listen on for HTTP clients http_address = "0.0.0.0:4161" ## address that will be registered with lookupd (defaults to the OS hostname) # broadcast_address = "" ## duration of time a producer will remain in the active list since its last ping inactive_producer_timeout = "300s" ## duration of time a producer will remain tombstoned if registration remains tombstone_lifetime = "45s" ================================================ FILE: coverage.sh ================================================ #!/bin/bash # Generate test coverage statistics for Go packages. # # Works around the fact that `go test -coverprofile` currently does not work # with multiple packages, see https://code.google.com/p/go/issues/detail?id=6909 # # Usage: coverage.sh [--html|--coveralls] # # --html Additionally create HTML report # --coveralls Push coverage statistics to coveralls.io # set -e workdir=.cover profile="$workdir/cover.out" mode=count generate_cover_data() { rm -rf "$workdir" mkdir "$workdir" for pkg in "$@"; do f="$workdir/$(echo $pkg | tr / -).cover" go test -covermode="$mode" -coverprofile="$f" "$pkg" done echo "mode: $mode" >"$profile" grep -h -v "^mode:" "$workdir"/*.cover >>"$profile" } show_html_report() { go tool cover -html="$profile" -o="$workdir"/coverage.html } show_csv_report() { go tool cover -func="$profile" -o="$workdir"/coverage.csv } push_to_coveralls() { echo "Pushing coverage statistics to coveralls.io" # ignore failure to push - it happens $GOPATH/bin/goveralls -coverprofile="$profile" \ -service=github \ -ignore="nsqadmin/bindata.go" || true } generate_cover_data $(go list ./... | grep -v /vendor/) show_csv_report case "$1" in "") ;; --html) show_html_report ;; --coveralls) push_to_coveralls ;; *) echo >&2 "error: invalid option: $1"; exit 1 ;; esac ================================================ FILE: dist.sh ================================================ #!/bin/bash # 1. commit to bump the version and update the changelog/readme # 2. tag that commit # 3. use dist.sh to produce tar.gz for all platforms # 4. aws s3 cp dist s3://bitly-downloads/nsq/ --recursive --exclude "*" --include "nsq-1.3.0*" --profile bitly --acl public-read # 5. docker manifest push nsqio/nsq:latest # 6. push to nsqio/master # 7. update the release metadata on github / upload the binaries # 8. update nsqio/nsqio.github.io/_posts/2014-03-01-installing.md # 9. send release announcement emails # 10. update IRC channel topic # 11. tweet set -e DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" rm -rf $DIR/dist/docker mkdir -p $DIR/dist/docker GOFLAGS='-ldflags="-s -w"' version=$(awk '/const Binary/ {print $NF}' < $DIR/internal/version/binary.go | sed 's/"//g') goversion=$(go version | awk '{print $3}') echo "... running tests" ./test.sh export GO111MODULE=on for target in "linux/amd64" "linux/arm64" "darwin/amd64" "darwin/arm64" "freebsd/amd64" "windows/amd64"; do os=${target%/*} arch=${target##*/} echo "... building v$version for $os/$arch" BUILD=$(mktemp -d ${TMPDIR:-/tmp}/nsq-XXXXX) TARGET="nsq-$version.$os-$arch.$goversion" GOOS=$os GOARCH=$arch CGO_ENABLED=0 \ make DESTDIR=$BUILD PREFIX=/$TARGET BLDFLAGS="$GOFLAGS" install pushd $BUILD sudo chown -R 0:0 $TARGET tar czvf $TARGET.tar.gz $TARGET mv $TARGET.tar.gz $DIR/dist popd make clean sudo rm -r $BUILD done rnd=$(LC_ALL=C tr -dc 'a-zA-Z0-9' < /dev/urandom | head -c10) docker buildx create --use --name nsq-$rnd docker buildx build .\ --tag nsqio/nsq:v$version \ --platform linux/amd64,linux/arm64 \ --output type=image,push=true if [[ ! $version == *"-"* ]]; then echo "Tagging nsqio/nsq:v$version as the latest release" shas=$(docker manifest inspect nsqio/nsq:v$version |\ grep digest | awk '{print $2}' | sed 's/[",]//g' | sed 's/^/nsqio\/nsq@/') docker manifest create --amend nsqio/nsq:latest $shas fi ================================================ FILE: fmt.sh ================================================ #!/bin/bash find . -name "*.go" | xargs goimports -w ================================================ FILE: go.mod ================================================ module github.com/nsqio/nsq go 1.17 require ( github.com/BurntSushi/toml v1.3.2 github.com/bitly/go-hostpool v0.1.0 github.com/bitly/timer_metrics v1.0.0 github.com/blang/semver v3.5.1+incompatible github.com/bmizerany/perks v0.0.0-20141205001514-d9a9656a3a4b github.com/golang/snappy v0.0.4 github.com/judwhite/go-svc v1.2.1 github.com/julienschmidt/httprouter v1.3.0 github.com/mreiferson/go-options v1.0.0 github.com/nsqio/go-diskqueue v1.1.0 github.com/nsqio/go-nsq v1.1.0 ) require ( github.com/stretchr/testify v1.9.0 // indirect golang.org/x/sys v0.10.0 // indirect ) replace github.com/judwhite/go-svc => github.com/mreiferson/go-svc v1.2.2-0.20210815184239-7a96e00010f6 ================================================ FILE: go.sum ================================================ github.com/BurntSushi/toml v1.3.2 h1:o7IhLm0Msx3BaB+n3Ag7L8EVlByGnpq14C4YWiu/gL8= github.com/BurntSushi/toml v1.3.2/go.mod h1:CxXYINrC8qIiEnFrOxCa7Jy5BFHlXnUU2pbicEuybxQ= github.com/bitly/go-hostpool v0.1.0 h1:XKmsF6k5el6xHG3WPJ8U0Ku/ye7njX7W81Ng7O2ioR0= github.com/bitly/go-hostpool v0.1.0/go.mod h1:4gOCgp6+NZnVqlKyZ/iBZFTAJKembaVENUpMkpg42fw= github.com/bitly/timer_metrics v1.0.0 h1:bbszVIl0vT5+/cdZx8L4KOQmM8mC/0y3EBICGSxyhCk= github.com/bitly/timer_metrics v1.0.0/go.mod h1:87z4/LSg3f++tMqZwZlsLwPuJu6xloyJ7Qm40NyEkLs= github.com/blang/semver v3.5.1+incompatible h1:cQNTCjp13qL8KC3Nbxr/y2Bqb63oX6wdnnjpJbkM4JQ= github.com/blang/semver v3.5.1+incompatible/go.mod h1:kRBLl5iJ+tD4TcOOxsy/0fnwebNt5EWlYSAyrTnjyyk= github.com/bmizerany/perks v0.0.0-20141205001514-d9a9656a3a4b h1:AP/Y7sqYicnjGDfD5VcY4CIfh1hRXBUavxrvELjTiOE= github.com/bmizerany/perks v0.0.0-20141205001514-d9a9656a3a4b/go.mod h1:ac9efd0D1fsDb3EJvhqgXRbFx7bs2wqZ10HQPeU8U/Q= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/golang/snappy v0.0.4 h1:yAGX7huGHXlcLOEtBnF4w7FQwA26wojNCwOYAEhLjQM= github.com/golang/snappy v0.0.4/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/julienschmidt/httprouter v1.3.0 h1:U0609e9tgbseu3rBINet9P48AI/D3oJs4dN7jwJOQ1U= github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8IZAc4RVcycCCAKdM= github.com/mreiferson/go-options v1.0.0 h1:RMLidydGlDWpL+lQTXo0bVIf/XT2CTq7AEJMoz5/VWs= github.com/mreiferson/go-options v1.0.0/go.mod h1:zHtCks/HQvOt8ATyfwVe3JJq2PPuImzXINPRTC03+9w= github.com/mreiferson/go-svc v1.2.2-0.20210815184239-7a96e00010f6 h1:NbuBXARvEXrYZ1SzN53ZpObeuwGhl1zvs/C+kzCggrQ= github.com/mreiferson/go-svc v1.2.2-0.20210815184239-7a96e00010f6/go.mod h1:mo/P2JNX8C07ywpP9YtO2gnBgnUiFTHqtsZekJrUuTk= github.com/nsqio/go-diskqueue v1.1.0 h1:r0dJ0DMXT3+2mOq+79cvCjnhoBxyGC2S9O+OjQrpe4Q= github.com/nsqio/go-diskqueue v1.1.0/go.mod h1:INuJIxl4ayUsyoNtHL5+9MFPDfSZ0zY93hNY6vhBRsI= github.com/nsqio/go-nsq v1.1.0 h1:PQg+xxiUjA7V+TLdXw7nVrJ5Jbl3sN86EhGCQj4+FYE= github.com/nsqio/go-nsq v1.1.0/go.mod h1:vKq36oyeVXgsS5Q8YEO7WghqidAVXQlcFxzQbQTuDEY= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.10.0 h1:SqMFp9UcQJZa+pmYuAKjd9xq1f0j5rLcDIk0mj4qAsA= golang.org/x/sys v0.10.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= ================================================ FILE: internal/app/float_array.go ================================================ package app import ( "fmt" "log" "sort" "strconv" "strings" ) type FloatArray []float64 func (a *FloatArray) Get() interface{} { return []float64(*a) } func (a *FloatArray) Set(param string) error { for _, s := range strings.Split(param, ",") { v, err := strconv.ParseFloat(s, 64) if err != nil { log.Fatalf("Could not parse: %s", s) return nil } *a = append(*a, v) } sort.Sort(*a) return nil } func (a FloatArray) Swap(i, j int) { a[i], a[j] = a[j], a[i] } func (a FloatArray) Less(i, j int) bool { return a[i] > a[j] } func (a FloatArray) Len() int { return len(a) } func (a *FloatArray) String() string { var s []string for _, v := range *a { s = append(s, fmt.Sprintf("%f", v)) } return strings.Join(s, ",") } ================================================ FILE: internal/app/string_array.go ================================================ package app import ( "strings" ) type StringArray []string func (a *StringArray) Get() interface{} { return []string(*a) } func (a *StringArray) Set(s string) error { *a = append(*a, s) return nil } func (a *StringArray) String() string { return strings.Join(*a, ",") } ================================================ FILE: internal/auth/authorizations.go ================================================ package auth import ( "crypto/tls" "errors" "fmt" "math/rand" "net/url" "regexp" "strings" "time" "github.com/nsqio/nsq/internal/http_api" ) type Authorization struct { Topic string `json:"topic"` Channels []string `json:"channels"` Permissions []string `json:"permissions"` } type State struct { TTL int `json:"ttl"` Authorizations []Authorization `json:"authorizations"` Identity string `json:"identity"` IdentityURL string `json:"identity_url"` Expires time.Time } func (a *Authorization) HasPermission(permission string) bool { for _, p := range a.Permissions { if permission == p { return true } } return false } func (a *Authorization) IsAllowed(topic, channel string) bool { if channel != "" { if !a.HasPermission("subscribe") { return false } } else { if !a.HasPermission("publish") { return false } } topicRegex := regexp.MustCompile(a.Topic) if !topicRegex.MatchString(topic) { return false } for _, c := range a.Channels { channelRegex := regexp.MustCompile(c) if channelRegex.MatchString(channel) { return true } } return false } func (a *State) IsAllowed(topic, channel string) bool { for _, aa := range a.Authorizations { if aa.IsAllowed(topic, channel) { return true } } return false } func (a *State) IsExpired() bool { return a.Expires.Before(time.Now()) } func QueryAnyAuthd(authd []string, remoteIP string, tlsEnabled bool, commonName string, authSecret string, clientTLSConfig *tls.Config, connectTimeout time.Duration, requestTimeout time.Duration, httpRequestMethod string) (*State, error) { var retErr error start := rand.Int() n := len(authd) for i := 0; i < n; i++ { a := authd[(i+start)%n] authState, err := QueryAuthd(a, remoteIP, tlsEnabled, commonName, authSecret, clientTLSConfig, connectTimeout, requestTimeout, httpRequestMethod) if err != nil { es := fmt.Sprintf("failed to auth against %s - %s", a, err) if retErr != nil { es = fmt.Sprintf("%s; %s", retErr, es) } retErr = errors.New(es) continue } return authState, nil } return nil, retErr } func QueryAuthd(authd string, remoteIP string, tlsEnabled bool, commonName string, authSecret string, clientTLSConfig *tls.Config, connectTimeout time.Duration, requestTimeout time.Duration, httpRequestMethod string) (*State, error) { var authState State v := url.Values{} v.Set("remote_ip", remoteIP) if tlsEnabled { v.Set("tls", "true") } else { v.Set("tls", "false") } v.Set("secret", authSecret) v.Set("common_name", commonName) var endpoint string if strings.Contains(authd, "://") { endpoint = authd } else { endpoint = fmt.Sprintf("http://%s/auth", authd) } client := http_api.NewClient(clientTLSConfig, connectTimeout, requestTimeout) if httpRequestMethod == "post" { if err := client.POSTV1(endpoint, v, &authState); err != nil { return nil, err } } else { endpoint = fmt.Sprintf("%s?%s", endpoint, v.Encode()) if err := client.GETV1(endpoint, &authState); err != nil { return nil, err } } // validation on response for _, auth := range authState.Authorizations { for _, p := range auth.Permissions { switch p { case "subscribe", "publish": default: return nil, fmt.Errorf("unknown permission %s", p) } } if _, err := regexp.Compile(auth.Topic); err != nil { return nil, fmt.Errorf("unable to compile topic %q %s", auth.Topic, err) } for _, channel := range auth.Channels { if _, err := regexp.Compile(channel); err != nil { return nil, fmt.Errorf("unable to compile channel %q %s", channel, err) } } } if authState.TTL <= 0 { return nil, fmt.Errorf("invalid TTL %d (must be >0)", authState.TTL) } authState.Expires = time.Now().Add(time.Duration(authState.TTL) * time.Second) return &authState, nil } ================================================ FILE: internal/clusterinfo/data.go ================================================ package clusterinfo import ( "fmt" "net" "net/url" "sort" "strconv" "strings" "sync" "github.com/blang/semver" "github.com/nsqio/nsq/internal/http_api" "github.com/nsqio/nsq/internal/lg" "github.com/nsqio/nsq/internal/stringy" ) type PartialErr interface { error Errors() []error } type ErrList []error func (l ErrList) Error() string { var es []string for _, e := range l { es = append(es, e.Error()) } return strings.Join(es, "\n") } func (l ErrList) Errors() []error { return l } type ClusterInfo struct { log lg.AppLogFunc client *http_api.Client } func New(log lg.AppLogFunc, client *http_api.Client) *ClusterInfo { return &ClusterInfo{ log: log, client: client, } } func (c *ClusterInfo) logf(f string, args ...interface{}) { if c.log != nil { c.log(lg.INFO, f, args...) } } // GetVersion returns a semver.Version object by querying /info func (c *ClusterInfo) GetVersion(addr string) (semver.Version, error) { endpoint := fmt.Sprintf("http://%s/info", addr) var resp struct { Version string `json:"version"` } err := c.client.GETV1(endpoint, &resp) if err != nil { return semver.Version{}, err } if resp.Version == "" { resp.Version = "unknown" } return semver.Parse(resp.Version) } // GetLookupdTopics returns a []string containing a union of all the topics // from all the given nsqlookupd func (c *ClusterInfo) GetLookupdTopics(lookupdHTTPAddrs []string) ([]string, error) { var topics []string var lock sync.Mutex var wg sync.WaitGroup var errs []error type respType struct { Topics []string `json:"topics"` } for _, addr := range lookupdHTTPAddrs { wg.Add(1) go func(addr string) { defer wg.Done() endpoint := fmt.Sprintf("http://%s/topics", addr) c.logf("CI: querying nsqlookupd %s", endpoint) var resp respType err := c.client.GETV1(endpoint, &resp) if err != nil { lock.Lock() errs = append(errs, err) lock.Unlock() return } lock.Lock() defer lock.Unlock() topics = append(topics, resp.Topics...) }(addr) } wg.Wait() if len(errs) == len(lookupdHTTPAddrs) { return nil, fmt.Errorf("failed to query any nsqlookupd: %s", ErrList(errs)) } topics = stringy.Uniq(topics) sort.Strings(topics) if len(errs) > 0 { return topics, ErrList(errs) } return topics, nil } // GetLookupdTopicChannels returns a []string containing a union of all the channels // from all the given lookupd for the given topic func (c *ClusterInfo) GetLookupdTopicChannels(topic string, lookupdHTTPAddrs []string) ([]string, error) { var channels []string var lock sync.Mutex var wg sync.WaitGroup var errs []error type respType struct { Channels []string `json:"channels"` } for _, addr := range lookupdHTTPAddrs { wg.Add(1) go func(addr string) { defer wg.Done() endpoint := fmt.Sprintf("http://%s/channels?topic=%s", addr, url.QueryEscape(topic)) c.logf("CI: querying nsqlookupd %s", endpoint) var resp respType err := c.client.GETV1(endpoint, &resp) if err != nil { lock.Lock() errs = append(errs, err) lock.Unlock() return } lock.Lock() defer lock.Unlock() channels = append(channels, resp.Channels...) }(addr) } wg.Wait() if len(errs) == len(lookupdHTTPAddrs) { return nil, fmt.Errorf("failed to query any nsqlookupd: %s", ErrList(errs)) } channels = stringy.Uniq(channels) sort.Strings(channels) if len(errs) > 0 { return channels, ErrList(errs) } return channels, nil } // GetLookupdProducers returns Producers of all the nsqd connected to the given lookupds func (c *ClusterInfo) GetLookupdProducers(lookupdHTTPAddrs []string) (Producers, error) { var producers []*Producer var lock sync.Mutex var wg sync.WaitGroup var errs []error producersByAddr := make(map[string]*Producer) maxVersion, _ := semver.Parse("0.0.0") type respType struct { Producers []*Producer `json:"producers"` } for _, addr := range lookupdHTTPAddrs { wg.Add(1) go func(addr string) { defer wg.Done() endpoint := fmt.Sprintf("http://%s/nodes", addr) c.logf("CI: querying nsqlookupd %s", endpoint) var resp respType err := c.client.GETV1(endpoint, &resp) if err != nil { lock.Lock() errs = append(errs, err) lock.Unlock() return } lock.Lock() defer lock.Unlock() for _, producer := range resp.Producers { key := producer.TCPAddress() p, ok := producersByAddr[key] if !ok { producersByAddr[key] = producer producers = append(producers, producer) if maxVersion.LT(producer.VersionObj) { maxVersion = producer.VersionObj } sort.Sort(producer.Topics) p = producer } p.RemoteAddresses = append(p.RemoteAddresses, fmt.Sprintf("%s/%s", addr, producer.Address())) } }(addr) } wg.Wait() if len(errs) == len(lookupdHTTPAddrs) { return nil, fmt.Errorf("failed to query any nsqlookupd: %s", ErrList(errs)) } for _, producer := range producersByAddr { if producer.VersionObj.LT(maxVersion) { producer.OutOfDate = true } } sort.Sort(ProducersByHost{producers}) if len(errs) > 0 { return producers, ErrList(errs) } return producers, nil } // GetLookupdTopicProducers returns Producers of all the nsqd for a given topic by // unioning the nodes returned from the given lookupd func (c *ClusterInfo) GetLookupdTopicProducers(topic string, lookupdHTTPAddrs []string) (Producers, error) { var producers Producers var lock sync.Mutex var wg sync.WaitGroup var errs []error type respType struct { Producers Producers `json:"producers"` } for _, addr := range lookupdHTTPAddrs { wg.Add(1) go func(addr string) { defer wg.Done() endpoint := fmt.Sprintf("http://%s/lookup?topic=%s", addr, url.QueryEscape(topic)) c.logf("CI: querying nsqlookupd %s", endpoint) var resp respType err := c.client.GETV1(endpoint, &resp) if err != nil { lock.Lock() errs = append(errs, err) lock.Unlock() return } lock.Lock() defer lock.Unlock() for _, p := range resp.Producers { for _, pp := range producers { if p.HTTPAddress() == pp.HTTPAddress() { goto skip } } producers = append(producers, p) skip: } }(addr) } wg.Wait() if len(errs) == len(lookupdHTTPAddrs) { return nil, fmt.Errorf("failed to query any nsqlookupd: %s", ErrList(errs)) } if len(errs) > 0 { return producers, ErrList(errs) } return producers, nil } // GetNSQDTopics returns a []string containing all the topics produced by the given nsqd func (c *ClusterInfo) GetNSQDTopics(nsqdHTTPAddrs []string) ([]string, error) { var topics []string var lock sync.Mutex var wg sync.WaitGroup var errs []error type respType struct { Topics []struct { Name string `json:"topic_name"` } `json:"topics"` } for _, addr := range nsqdHTTPAddrs { wg.Add(1) go func(addr string) { defer wg.Done() endpoint := fmt.Sprintf("http://%s/stats?format=json", addr) c.logf("CI: querying nsqd %s", endpoint) var resp respType err := c.client.GETV1(endpoint, &resp) if err != nil { lock.Lock() errs = append(errs, err) lock.Unlock() return } lock.Lock() defer lock.Unlock() for _, topic := range resp.Topics { topics = stringy.Add(topics, topic.Name) } }(addr) } wg.Wait() if len(errs) == len(nsqdHTTPAddrs) { return nil, fmt.Errorf("failed to query any nsqd: %s", ErrList(errs)) } sort.Strings(topics) if len(errs) > 0 { return topics, ErrList(errs) } return topics, nil } // GetNSQDProducers returns Producers of all the given nsqd func (c *ClusterInfo) GetNSQDProducers(nsqdHTTPAddrs []string) (Producers, error) { var producers Producers var lock sync.Mutex var wg sync.WaitGroup var errs []error type infoRespType struct { Version string `json:"version"` BroadcastAddress string `json:"broadcast_address"` Hostname string `json:"hostname"` HTTPPort int `json:"http_port"` TCPPort int `json:"tcp_port"` TopologyZone string `json:"topology_zone,omitempty"` TopologyRegion string `json:"topology_region,omitempty"` } type statsRespType struct { Topics []struct { Name string `json:"topic_name"` } `json:"topics"` } for _, addr := range nsqdHTTPAddrs { wg.Add(1) go func(addr string) { defer wg.Done() endpoint := fmt.Sprintf("http://%s/info", addr) c.logf("CI: querying nsqd %s", endpoint) var infoResp infoRespType err := c.client.GETV1(endpoint, &infoResp) if err != nil { lock.Lock() errs = append(errs, err) lock.Unlock() return } endpoint = fmt.Sprintf("http://%s/stats?format=json&include_clients=false", addr) c.logf("CI: querying nsqd %s", endpoint) var statsResp statsRespType err = c.client.GETV1(endpoint, &statsResp) if err != nil { lock.Lock() errs = append(errs, err) lock.Unlock() return } var producerTopics ProducerTopics for _, t := range statsResp.Topics { producerTopics = append(producerTopics, ProducerTopic{Topic: t.Name}) } version, err := semver.Parse(infoResp.Version) if err != nil { version, _ = semver.Parse("0.0.0") } lock.Lock() defer lock.Unlock() producers = append(producers, &Producer{ Version: infoResp.Version, VersionObj: version, BroadcastAddress: infoResp.BroadcastAddress, Hostname: infoResp.Hostname, HTTPPort: infoResp.HTTPPort, TCPPort: infoResp.TCPPort, Topics: producerTopics, TopologyZone: infoResp.TopologyZone, TopologyRegion: infoResp.TopologyRegion, }) }(addr) } wg.Wait() if len(errs) == len(nsqdHTTPAddrs) { return nil, fmt.Errorf("failed to query any nsqd: %s", ErrList(errs)) } if len(errs) > 0 { return producers, ErrList(errs) } return producers, nil } // GetNSQDTopicProducers returns Producers containing the addresses of all the nsqd // that produce the given topic func (c *ClusterInfo) GetNSQDTopicProducers(topic string, nsqdHTTPAddrs []string) (Producers, error) { var producers Producers var lock sync.Mutex var wg sync.WaitGroup var errs []error type infoRespType struct { Version string `json:"version"` BroadcastAddress string `json:"broadcast_address"` Hostname string `json:"hostname"` HTTPPort int `json:"http_port"` TCPPort int `json:"tcp_port"` TopologyZone string `json:"topology_zone,omitempty"` TopologyRegion string `json:"topology_region,omitempty"` } type statsRespType struct { Topics []struct { Name string `json:"topic_name"` } `json:"topics"` } for _, addr := range nsqdHTTPAddrs { wg.Add(1) go func(addr string) { defer wg.Done() endpoint := fmt.Sprintf("http://%s/stats?format=json&topic=%s&include_clients=false", addr, url.QueryEscape(topic)) c.logf("CI: querying nsqd %s", endpoint) var statsResp statsRespType err := c.client.GETV1(endpoint, &statsResp) if err != nil { lock.Lock() errs = append(errs, err) lock.Unlock() return } var producerTopics ProducerTopics for _, t := range statsResp.Topics { producerTopics = append(producerTopics, ProducerTopic{Topic: t.Name}) } for _, t := range statsResp.Topics { if t.Name == topic { endpoint := fmt.Sprintf("http://%s/info", addr) c.logf("CI: querying nsqd %s", endpoint) var infoResp infoRespType err := c.client.GETV1(endpoint, &infoResp) if err != nil { lock.Lock() errs = append(errs, err) lock.Unlock() return } version, err := semver.Parse(infoResp.Version) if err != nil { version, _ = semver.Parse("0.0.0") } // if BroadcastAddress/HTTPPort are missing, use the values from `addr` for // backwards compatibility if infoResp.BroadcastAddress == "" { var p string infoResp.BroadcastAddress, p, _ = net.SplitHostPort(addr) infoResp.HTTPPort, _ = strconv.Atoi(p) } if infoResp.Hostname == "" { infoResp.Hostname, _, _ = net.SplitHostPort(addr) } lock.Lock() producers = append(producers, &Producer{ Version: infoResp.Version, VersionObj: version, BroadcastAddress: infoResp.BroadcastAddress, Hostname: infoResp.Hostname, HTTPPort: infoResp.HTTPPort, TCPPort: infoResp.TCPPort, Topics: producerTopics, TopologyZone: infoResp.TopologyZone, TopologyRegion: infoResp.TopologyRegion, }) lock.Unlock() return } } }(addr) } wg.Wait() if len(errs) == len(nsqdHTTPAddrs) { return nil, fmt.Errorf("failed to query any nsqd: %s", ErrList(errs)) } if len(errs) > 0 { return producers, ErrList(errs) } return producers, nil } // GetNSQDStats returns aggregate topic and channel stats from the given Producers // // if selectedChannel is empty, this will return stats for topic/channel // if selectedTopic is empty, this will return stats for *all* topic/channels // if includeClients is false, this will *not* return client stats for channels // and the ChannelStats dict will be keyed by topic + ':' + channel func (c *ClusterInfo) GetNSQDStats(producers Producers, selectedTopic string, selectedChannel string, includeClients bool) ([]*TopicStats, map[string]*ChannelStats, error) { var lock sync.Mutex var wg sync.WaitGroup var topicStatsList TopicStatsList var errs []error channelStatsMap := make(map[string]*ChannelStats) type respType struct { Topics []*TopicStats `json:"topics"` } for _, p := range producers { wg.Add(1) go func(p *Producer) { defer wg.Done() addr := p.HTTPAddress() endpoint := fmt.Sprintf("http://%s/stats?format=json", addr) if selectedTopic != "" { endpoint += "&topic=" + url.QueryEscape(selectedTopic) if selectedChannel != "" { endpoint += "&channel=" + url.QueryEscape(selectedChannel) } } if !includeClients { endpoint += "&include_clients=false" } c.logf("CI: querying nsqd %s", endpoint) var resp respType err := c.client.GETV1(endpoint, &resp) if err != nil { lock.Lock() errs = append(errs, err) lock.Unlock() return } lock.Lock() defer lock.Unlock() for _, topic := range resp.Topics { topic.Node = addr topic.Hostname = p.Hostname topic.MemoryDepth = topic.Depth - topic.BackendDepth topic.DeliveryMsgCount = topic.ZoneLocalMsgCount + topic.RegionLocalMsgCount + topic.GlobalMsgCount if selectedTopic != "" && topic.TopicName != selectedTopic { continue } topicStatsList = append(topicStatsList, topic) for _, channel := range topic.Channels { channel.Node = addr channel.Hostname = p.Hostname channel.TopicName = topic.TopicName channel.MemoryDepth = channel.Depth - channel.BackendDepth channel.DeliveryMsgCount = channel.ZoneLocalMsgCount + channel.RegionLocalMsgCount + channel.GlobalMsgCount key := channel.ChannelName if selectedTopic == "" { key = fmt.Sprintf("%s:%s", topic.TopicName, channel.ChannelName) } channelStats, ok := channelStatsMap[key] if !ok { channelStats = &ChannelStats{ Node: addr, TopicName: topic.TopicName, ChannelName: channel.ChannelName, } channelStatsMap[key] = channelStats } for _, c := range channel.Clients { c.Node = addr c.NodeTopologyRegion = p.TopologyRegion c.NodeTopologyZone = p.TopologyZone } channelStats.Add(channel) } } }(p) } wg.Wait() if len(errs) == len(producers) { return nil, nil, fmt.Errorf("failed to query any nsqd: %s", ErrList(errs)) } sort.Sort(TopicStatsByHost{topicStatsList}) if len(errs) > 0 { return topicStatsList, channelStatsMap, ErrList(errs) } return topicStatsList, channelStatsMap, nil } // TombstoneNodeForTopic tombstones the given node for the given topic on all the given nsqlookupd // and deletes the topic from the node func (c *ClusterInfo) TombstoneNodeForTopic(topic string, node string, lookupdHTTPAddrs []string) error { var errs []error // tombstone the topic on all the lookupds qs := fmt.Sprintf("topic=%s&node=%s", url.QueryEscape(topic), url.QueryEscape(node)) err := c.nsqlookupdPOST(lookupdHTTPAddrs, "topic/tombstone", qs) if err != nil { pe, ok := err.(PartialErr) if !ok { return err } errs = append(errs, pe.Errors()...) } producers, err := c.GetNSQDProducers([]string{node}) if err != nil { pe, ok := err.(PartialErr) if !ok { return err } errs = append(errs, pe.Errors()...) } // delete the topic on the producer qs = fmt.Sprintf("topic=%s", url.QueryEscape(topic)) err = c.producersPOST(producers, "topic/delete", qs) if err != nil { pe, ok := err.(PartialErr) if !ok { return err } errs = append(errs, pe.Errors()...) } if len(errs) > 0 { return ErrList(errs) } return nil } func (c *ClusterInfo) CreateTopicChannel(topicName string, channelName string, lookupdHTTPAddrs []string) error { var errs []error // create the topic on all the nsqlookupd qs := fmt.Sprintf("topic=%s", url.QueryEscape(topicName)) err := c.nsqlookupdPOST(lookupdHTTPAddrs, "topic/create", qs) if err != nil { pe, ok := err.(PartialErr) if !ok { return err } errs = append(errs, pe.Errors()...) } if len(channelName) > 0 { qs := fmt.Sprintf("topic=%s&channel=%s", url.QueryEscape(topicName), url.QueryEscape(channelName)) // create the channel on all the nsqlookupd err := c.nsqlookupdPOST(lookupdHTTPAddrs, "channel/create", qs) if err != nil { pe, ok := err.(PartialErr) if !ok { return err } errs = append(errs, pe.Errors()...) } // create the channel on all the nsqd that produce the topic producers, err := c.GetLookupdTopicProducers(topicName, lookupdHTTPAddrs) if err != nil { pe, ok := err.(PartialErr) if !ok { return err } errs = append(errs, pe.Errors()...) } err = c.producersPOST(producers, "channel/create", qs) if err != nil { pe, ok := err.(PartialErr) if !ok { return err } errs = append(errs, pe.Errors()...) } } if len(errs) > 0 { return ErrList(errs) } return nil } func (c *ClusterInfo) DeleteTopic(topicName string, lookupdHTTPAddrs []string, nsqdHTTPAddrs []string) error { var errs []error // for topic removal, you need to get all the producers _first_ producers, err := c.GetTopicProducers(topicName, lookupdHTTPAddrs, nsqdHTTPAddrs) if err != nil { pe, ok := err.(PartialErr) if !ok { return err } errs = append(errs, pe.Errors()...) } qs := fmt.Sprintf("topic=%s", url.QueryEscape(topicName)) // remove the topic from all the nsqlookupd err = c.nsqlookupdPOST(lookupdHTTPAddrs, "topic/delete", qs) if err != nil { pe, ok := err.(PartialErr) if !ok { return err } errs = append(errs, pe.Errors()...) } // remove the topic from all the nsqd that produce this topic err = c.producersPOST(producers, "topic/delete", qs) if err != nil { pe, ok := err.(PartialErr) if !ok { return err } errs = append(errs, pe.Errors()...) } if len(errs) > 0 { return ErrList(errs) } return nil } func (c *ClusterInfo) DeleteChannel(topicName string, channelName string, lookupdHTTPAddrs []string, nsqdHTTPAddrs []string) error { var errs []error producers, err := c.GetTopicProducers(topicName, lookupdHTTPAddrs, nsqdHTTPAddrs) if err != nil { pe, ok := err.(PartialErr) if !ok { return err } errs = append(errs, pe.Errors()...) } qs := fmt.Sprintf("topic=%s&channel=%s", url.QueryEscape(topicName), url.QueryEscape(channelName)) // remove the channel from all the nsqlookupd err = c.nsqlookupdPOST(lookupdHTTPAddrs, "channel/delete", qs) if err != nil { pe, ok := err.(PartialErr) if !ok { return err } errs = append(errs, pe.Errors()...) } // remove the channel from all the nsqd that produce this topic err = c.producersPOST(producers, "channel/delete", qs) if err != nil { pe, ok := err.(PartialErr) if !ok { return err } errs = append(errs, pe.Errors()...) } if len(errs) > 0 { return ErrList(errs) } return nil } func (c *ClusterInfo) PauseTopic(topicName string, lookupdHTTPAddrs []string, nsqdHTTPAddrs []string) error { qs := fmt.Sprintf("topic=%s", url.QueryEscape(topicName)) return c.actionHelper(topicName, lookupdHTTPAddrs, nsqdHTTPAddrs, "topic/pause", qs) } func (c *ClusterInfo) UnPauseTopic(topicName string, lookupdHTTPAddrs []string, nsqdHTTPAddrs []string) error { qs := fmt.Sprintf("topic=%s", url.QueryEscape(topicName)) return c.actionHelper(topicName, lookupdHTTPAddrs, nsqdHTTPAddrs, "topic/unpause", qs) } func (c *ClusterInfo) PauseChannel(topicName string, channelName string, lookupdHTTPAddrs []string, nsqdHTTPAddrs []string) error { qs := fmt.Sprintf("topic=%s&channel=%s", url.QueryEscape(topicName), url.QueryEscape(channelName)) return c.actionHelper(topicName, lookupdHTTPAddrs, nsqdHTTPAddrs, "channel/pause", qs) } func (c *ClusterInfo) UnPauseChannel(topicName string, channelName string, lookupdHTTPAddrs []string, nsqdHTTPAddrs []string) error { qs := fmt.Sprintf("topic=%s&channel=%s", url.QueryEscape(topicName), url.QueryEscape(channelName)) return c.actionHelper(topicName, lookupdHTTPAddrs, nsqdHTTPAddrs, "channel/unpause", qs) } func (c *ClusterInfo) EmptyTopic(topicName string, lookupdHTTPAddrs []string, nsqdHTTPAddrs []string) error { qs := fmt.Sprintf("topic=%s", url.QueryEscape(topicName)) return c.actionHelper(topicName, lookupdHTTPAddrs, nsqdHTTPAddrs, "topic/empty", qs) } func (c *ClusterInfo) EmptyChannel(topicName string, channelName string, lookupdHTTPAddrs []string, nsqdHTTPAddrs []string) error { qs := fmt.Sprintf("topic=%s&channel=%s", url.QueryEscape(topicName), url.QueryEscape(channelName)) return c.actionHelper(topicName, lookupdHTTPAddrs, nsqdHTTPAddrs, "channel/empty", qs) } func (c *ClusterInfo) actionHelper(topicName string, lookupdHTTPAddrs []string, nsqdHTTPAddrs []string, uri string, qs string) error { var errs []error producers, err := c.GetTopicProducers(topicName, lookupdHTTPAddrs, nsqdHTTPAddrs) if err != nil { pe, ok := err.(PartialErr) if !ok { return err } errs = append(errs, pe.Errors()...) } err = c.producersPOST(producers, uri, qs) if err != nil { pe, ok := err.(PartialErr) if !ok { return err } errs = append(errs, pe.Errors()...) } if len(errs) > 0 { return ErrList(errs) } return nil } func (c *ClusterInfo) GetProducers(lookupdHTTPAddrs []string, nsqdHTTPAddrs []string) (Producers, error) { if len(lookupdHTTPAddrs) != 0 { return c.GetLookupdProducers(lookupdHTTPAddrs) } return c.GetNSQDProducers(nsqdHTTPAddrs) } func (c *ClusterInfo) GetTopicProducers(topicName string, lookupdHTTPAddrs []string, nsqdHTTPAddrs []string) (Producers, error) { if len(lookupdHTTPAddrs) != 0 { return c.GetLookupdTopicProducers(topicName, lookupdHTTPAddrs) } return c.GetNSQDTopicProducers(topicName, nsqdHTTPAddrs) } func (c *ClusterInfo) nsqlookupdPOST(addrs []string, uri string, qs string) error { var errs []error for _, addr := range addrs { endpoint := fmt.Sprintf("http://%s/%s?%s", addr, uri, qs) c.logf("CI: querying nsqlookupd %s", endpoint) err := c.client.POSTV1(endpoint, nil, nil) if err != nil { errs = append(errs, err) } } if len(errs) > 0 { return ErrList(errs) } return nil } func (c *ClusterInfo) producersPOST(pl Producers, uri string, qs string) error { var errs []error for _, p := range pl { endpoint := fmt.Sprintf("http://%s/%s?%s", p.HTTPAddress(), uri, qs) c.logf("CI: querying nsqd %s", endpoint) err := c.client.POSTV1(endpoint, nil, nil) if err != nil { errs = append(errs, err) } } if len(errs) > 0 { return ErrList(errs) } return nil } ================================================ FILE: internal/clusterinfo/producer_test.go ================================================ package clusterinfo import "testing" func TestHostNameAddresses(t *testing.T) { p := &Producer{ BroadcastAddress: "host.domain.com", TCPPort: 4150, HTTPPort: 4151, } if p.HTTPAddress() != "host.domain.com:4151" { t.Errorf("Incorrect HTTPAddress: %s", p.HTTPAddress()) } if p.TCPAddress() != "host.domain.com:4150" { t.Errorf("Incorrect TCPAddress: %s", p.TCPAddress()) } } func TestIPv4Addresses(t *testing.T) { p := &Producer{ BroadcastAddress: "192.168.1.17", TCPPort: 4150, HTTPPort: 4151, } if p.HTTPAddress() != "192.168.1.17:4151" { t.Errorf("Incorrect IPv4 HTTPAddress: %s", p.HTTPAddress()) } if p.TCPAddress() != "192.168.1.17:4150" { t.Errorf("Incorrect IPv4 TCPAddress: %s", p.TCPAddress()) } } func TestIPv6Addresses(t *testing.T) { p := &Producer{ BroadcastAddress: "fd4a:622f:d2f2::1", TCPPort: 4150, HTTPPort: 4151, } if p.HTTPAddress() != "[fd4a:622f:d2f2::1]:4151" { t.Errorf("Incorrect IPv6 HTTPAddress: %s", p.HTTPAddress()) } if p.TCPAddress() != "[fd4a:622f:d2f2::1]:4150" { t.Errorf("Incorrect IPv6 TCPAddress: %s", p.TCPAddress()) } } ================================================ FILE: internal/clusterinfo/types.go ================================================ package clusterinfo import ( "encoding/json" "net" "sort" "strconv" "time" "github.com/blang/semver" "github.com/nsqio/nsq/internal/quantile" ) type ProducerTopic struct { Topic string `json:"topic"` Tombstoned bool `json:"tombstoned"` } type ProducerTopics []ProducerTopic func (pt ProducerTopics) Len() int { return len(pt) } func (pt ProducerTopics) Swap(i, j int) { pt[i], pt[j] = pt[j], pt[i] } func (pt ProducerTopics) Less(i, j int) bool { return pt[i].Topic < pt[j].Topic } type Producer struct { RemoteAddresses []string `json:"remote_addresses"` RemoteAddress string `json:"remote_address"` Hostname string `json:"hostname"` BroadcastAddress string `json:"broadcast_address"` TCPPort int `json:"tcp_port"` HTTPPort int `json:"http_port"` Version string `json:"version"` TopologyZone string `json:"topology_zone,omitempty"` TopologyRegion string `json:"topology_region,omitempty"` VersionObj semver.Version `json:"-"` Topics ProducerTopics `json:"topics"` OutOfDate bool `json:"out_of_date"` } // UnmarshalJSON implements json.Unmarshaler and postprocesses of ProducerTopics and VersionObj func (p *Producer) UnmarshalJSON(b []byte) error { var r struct { RemoteAddress string `json:"remote_address"` Hostname string `json:"hostname"` BroadcastAddress string `json:"broadcast_address"` TCPPort int `json:"tcp_port"` HTTPPort int `json:"http_port"` Version string `json:"version"` Topics []string `json:"topics"` Tombstoned []bool `json:"tombstones"` TopologyZone string `json:"topology_zone,omitempty"` TopologyRegion string `json:"topology_region,omitempty"` } if err := json.Unmarshal(b, &r); err != nil { return err } *p = Producer{ RemoteAddress: r.RemoteAddress, Hostname: r.Hostname, BroadcastAddress: r.BroadcastAddress, TCPPort: r.TCPPort, HTTPPort: r.HTTPPort, Version: r.Version, TopologyZone: r.TopologyZone, TopologyRegion: r.TopologyRegion, } for i, t := range r.Topics { p.Topics = append(p.Topics, ProducerTopic{Topic: t, Tombstoned: r.Tombstoned[i]}) } version, err := semver.Parse(p.Version) if err != nil { version, _ = semver.Parse("0.0.0") } p.VersionObj = version return nil } func (p *Producer) Address() string { if p.RemoteAddress == "" { return "N/A" } return p.RemoteAddress } func (p *Producer) HTTPAddress() string { return net.JoinHostPort(p.BroadcastAddress, strconv.Itoa(p.HTTPPort)) } func (p *Producer) TCPAddress() string { return net.JoinHostPort(p.BroadcastAddress, strconv.Itoa(p.TCPPort)) } // IsInconsistent checks for cases where an unexpected number of nsqd connections are // reporting the same information to nsqlookupd (ie: multiple instances are using the // same broadcast address), or cases where some nsqd are not reporting to all nsqlookupd. func (p *Producer) IsInconsistent(numLookupd int) bool { return len(p.RemoteAddresses) != numLookupd } type TopicStats struct { Node string `json:"node"` Hostname string `json:"hostname"` TopicName string `json:"topic_name"` Depth int64 `json:"depth"` MemoryDepth int64 `json:"memory_depth"` BackendDepth int64 `json:"backend_depth"` MessageCount int64 `json:"message_count"` DeliveryMsgCount int64 `json:"delivery_msg_count"` ZoneLocalMsgCount int64 `json:"zone_local_msg_count,omitempty"` RegionLocalMsgCount int64 `json:"region_local_msg_count,omitempty"` GlobalMsgCount int64 `json:"global_msg_count,omitempty"` NodeStats []*TopicStats `json:"nodes"` Channels []*ChannelStats `json:"channels"` Paused bool `json:"paused"` E2eProcessingLatency *quantile.E2eProcessingLatencyAggregate `json:"e2e_processing_latency"` } func (t *TopicStats) Add(a *TopicStats) { t.Node = "*" t.Depth += a.Depth t.MemoryDepth += a.MemoryDepth t.BackendDepth += a.BackendDepth t.MessageCount += a.MessageCount t.DeliveryMsgCount += a.DeliveryMsgCount t.ZoneLocalMsgCount += a.ZoneLocalMsgCount t.RegionLocalMsgCount += a.RegionLocalMsgCount t.GlobalMsgCount += a.GlobalMsgCount if a.Paused { t.Paused = a.Paused } for _, aChannelStats := range a.Channels { found := false for _, channelStats := range t.Channels { if aChannelStats.ChannelName == channelStats.ChannelName { found = true channelStats.Add(aChannelStats) } } if !found { t.Channels = append(t.Channels, aChannelStats) } } t.NodeStats = append(t.NodeStats, a) sort.Sort(TopicStatsByHost{t.NodeStats}) if t.E2eProcessingLatency == nil { t.E2eProcessingLatency = &quantile.E2eProcessingLatencyAggregate{ Addr: t.Node, Topic: t.TopicName, } } t.E2eProcessingLatency.Add(a.E2eProcessingLatency) } type ChannelStats struct { Node string `json:"node"` Hostname string `json:"hostname"` TopicName string `json:"topic_name"` ChannelName string `json:"channel_name"` Depth int64 `json:"depth"` MemoryDepth int64 `json:"memory_depth"` BackendDepth int64 `json:"backend_depth"` InFlightCount int64 `json:"in_flight_count"` DeferredCount int64 `json:"deferred_count"` RequeueCount int64 `json:"requeue_count"` TimeoutCount int64 `json:"timeout_count"` MessageCount int64 `json:"message_count"` DeliveryMsgCount int64 `json:"delivery_msg_count,omitempty"` ZoneLocalMsgCount int64 `json:"zone_local_msg_count,omitempty"` RegionLocalMsgCount int64 `json:"region_local_msg_count,omitempty"` GlobalMsgCount int64 `json:"global_msg_count,omitempty"` ClientCount int `json:"client_count"` Selected bool `json:"-"` NodeStats []*ChannelStats `json:"nodes"` Clients []*ClientStats `json:"clients"` Paused bool `json:"paused"` E2eProcessingLatency *quantile.E2eProcessingLatencyAggregate `json:"e2e_processing_latency"` } func (c *ChannelStats) Add(a *ChannelStats) { c.Node = "*" c.Depth += a.Depth c.MemoryDepth += a.MemoryDepth c.BackendDepth += a.BackendDepth c.InFlightCount += a.InFlightCount c.DeferredCount += a.DeferredCount c.RequeueCount += a.RequeueCount c.TimeoutCount += a.TimeoutCount c.MessageCount += a.MessageCount c.DeliveryMsgCount += a.DeliveryMsgCount c.ZoneLocalMsgCount += a.ZoneLocalMsgCount c.RegionLocalMsgCount += a.RegionLocalMsgCount c.GlobalMsgCount += a.GlobalMsgCount c.ClientCount += a.ClientCount if a.Paused { c.Paused = a.Paused } c.NodeStats = append(c.NodeStats, a) sort.Sort(ChannelStatsByHost{c.NodeStats}) if c.E2eProcessingLatency == nil { c.E2eProcessingLatency = &quantile.E2eProcessingLatencyAggregate{ Addr: c.Node, Topic: c.TopicName, Channel: c.ChannelName, } } c.E2eProcessingLatency.Add(a.E2eProcessingLatency) c.Clients = append(c.Clients, a.Clients...) sort.Sort(ClientsByHost{c.Clients}) } type ClientStats struct { Node string `json:"node"` RemoteAddress string `json:"remote_address"` Version string `json:"version"` ClientID string `json:"client_id"` Hostname string `json:"hostname"` UserAgent string `json:"user_agent"` ConnectTs int64 `json:"connect_ts"` ConnectedDuration time.Duration `json:"connected"` InFlightCount int `json:"in_flight_count"` ReadyCount int `json:"ready_count"` FinishCount int64 `json:"finish_count"` RequeueCount int64 `json:"requeue_count"` MessageCount int64 `json:"message_count"` SampleRate int32 `json:"sample_rate"` Deflate bool `json:"deflate"` Snappy bool `json:"snappy"` Authed bool `json:"authed"` AuthIdentity string `json:"auth_identity"` AuthIdentityURL string `json:"auth_identity_url"` NodeTopologyRegion string `json:"node_topology_region,omitempty"` NodeTopologyZone string `json:"node_topology_zone,omitempty"` TopologyRegion string `json:"topology_region,omitempty"` TopologyZone string `json:"topology_zone,omitempty"` TLS bool `json:"tls"` CipherSuite string `json:"tls_cipher_suite"` TLSVersion string `json:"tls_version"` TLSNegotiatedProtocol string `json:"tls_negotiated_protocol"` TLSNegotiatedProtocolIsMutual bool `json:"tls_negotiated_protocol_is_mutual"` } // UnmarshalJSON implements json.Unmarshaler and postprocesses ConnectedDuration func (s *ClientStats) UnmarshalJSON(b []byte) error { type locaClientStats ClientStats // re-typed to prevent recursion from json.Unmarshal var ss locaClientStats if err := json.Unmarshal(b, &ss); err != nil { return err } *s = ClientStats(ss) s.ConnectedDuration = time.Now().Truncate(time.Second).Sub(time.Unix(s.ConnectTs, 0)) return nil } func (s *ClientStats) HasUserAgent() bool { return s.UserAgent != "" } func (s *ClientStats) HasSampleRate() bool { return s.SampleRate > 0 } type ChannelStatsList []*ChannelStats func (c ChannelStatsList) Len() int { return len(c) } func (c ChannelStatsList) Swap(i, j int) { c[i], c[j] = c[j], c[i] } type ChannelStatsByHost struct { ChannelStatsList } func (c ChannelStatsByHost) Less(i, j int) bool { return c.ChannelStatsList[i].Hostname < c.ChannelStatsList[j].Hostname } type ClientStatsList []*ClientStats func (c ClientStatsList) Len() int { return len(c) } func (c ClientStatsList) Swap(i, j int) { c[i], c[j] = c[j], c[i] } type ClientsByHost struct { ClientStatsList } func (c ClientsByHost) Less(i, j int) bool { return c.ClientStatsList[i].Hostname < c.ClientStatsList[j].Hostname } type ClientStatsByNodeTopology struct { ClientStatsList } func (c ClientStatsByNodeTopology) Less(i, j int) bool { // if its the same node, sort by topology if c.ClientStatsList[i].Node == c.ClientStatsList[j].Node { region := c.ClientStatsList[i].NodeTopologyRegion zone := c.ClientStatsList[i].NodeTopologyZone switch { case c.ClientStatsList[i].TopologyRegion == region && c.ClientStatsList[i].TopologyZone == zone: return true case c.ClientStatsList[j].TopologyRegion == region && c.ClientStatsList[j].TopologyZone == zone: return false case c.ClientStatsList[i].TopologyRegion == region: return true case c.ClientStatsList[j].TopologyRegion == region: return false default: if c.ClientStatsList[i].TopologyRegion == c.ClientStatsList[j].TopologyRegion { return c.ClientStatsList[i].TopologyZone < c.ClientStatsList[j].TopologyZone } return c.ClientStatsList[i].TopologyRegion < c.ClientStatsList[j].TopologyRegion } } return c.ClientStatsList[i].Node < c.ClientStatsList[j].Node } type TopicStatsList []*TopicStats func (t TopicStatsList) Len() int { return len(t) } func (t TopicStatsList) Swap(i, j int) { t[i], t[j] = t[j], t[i] } type TopicStatsByHost struct { TopicStatsList } func (c TopicStatsByHost) Less(i, j int) bool { return c.TopicStatsList[i].Hostname < c.TopicStatsList[j].Hostname } type Producers []*Producer func (t Producers) Len() int { return len(t) } func (t Producers) Swap(i, j int) { t[i], t[j] = t[j], t[i] } func (t Producers) HTTPAddrs() []string { var addrs []string for _, p := range t { addrs = append(addrs, p.HTTPAddress()) } return addrs } func (t Producers) Search(needle string) *Producer { for _, producer := range t { if needle == producer.HTTPAddress() { return producer } } return nil } type ProducersByHost struct { Producers } func (c ProducersByHost) Less(i, j int) bool { return c.Producers[i].Hostname < c.Producers[j].Hostname } ================================================ FILE: internal/dirlock/dirlock.go ================================================ //go:build !windows && !illumos // +build !windows,!illumos package dirlock import ( "fmt" "os" "syscall" ) type DirLock struct { dir string f *os.File } func New(dir string) *DirLock { return &DirLock{ dir: dir, } } func (l *DirLock) Lock() error { f, err := os.Open(l.dir) if err != nil { return err } l.f = f err = syscall.Flock(int(f.Fd()), syscall.LOCK_EX|syscall.LOCK_NB) if err != nil { return fmt.Errorf("cannot flock directory %s - %s (possibly in use by another instance of nsqd)", l.dir, err) } return nil } func (l *DirLock) Unlock() error { defer l.f.Close() return syscall.Flock(int(l.f.Fd()), syscall.LOCK_UN) } ================================================ FILE: internal/dirlock/dirlock_illumos.go ================================================ //go:build illumos // +build illumos package dirlock type DirLock struct { dir string } func New(dir string) *DirLock { return &DirLock{ dir: dir, } } func (l *DirLock) Lock() error { return nil } func (l *DirLock) Unlock() error { return nil } ================================================ FILE: internal/dirlock/dirlock_windows.go ================================================ //go:build windows // +build windows package dirlock type DirLock struct { dir string } func New(dir string) *DirLock { return &DirLock{ dir: dir, } } func (l *DirLock) Lock() error { return nil } func (l *DirLock) Unlock() error { return nil } ================================================ FILE: internal/http_api/api_request.go ================================================ package http_api import ( "bytes" "crypto/tls" "encoding/json" "fmt" "io" "net" "net/http" "net/url" "strconv" "strings" "time" ) // A custom http.Transport with support for deadline timeouts func NewDeadlineTransport(connectTimeout time.Duration, requestTimeout time.Duration) *http.Transport { // arbitrary values copied from http.DefaultTransport transport := &http.Transport{ DialContext: (&net.Dialer{ Timeout: connectTimeout, KeepAlive: 30 * time.Second, DualStack: true, }).DialContext, ResponseHeaderTimeout: requestTimeout, MaxIdleConns: 100, IdleConnTimeout: 90 * time.Second, TLSHandshakeTimeout: 10 * time.Second, } return transport } type Client struct { c *http.Client } func NewClient(tlsConfig *tls.Config, connectTimeout time.Duration, requestTimeout time.Duration) *Client { transport := NewDeadlineTransport(connectTimeout, requestTimeout) transport.TLSClientConfig = tlsConfig return &Client{ c: &http.Client{ Transport: transport, Timeout: requestTimeout, }, } } // GETV1 is a helper function to perform a V1 HTTP request // and parse our NSQ daemon's expected response format, with deadlines. func (c *Client) GETV1(endpoint string, v interface{}) error { retry: req, err := http.NewRequest("GET", endpoint, nil) if err != nil { return err } req.Header.Add("Accept", "application/vnd.nsq; version=1.0") resp, err := c.c.Do(req) if err != nil { return err } body, err := io.ReadAll(resp.Body) resp.Body.Close() if err != nil { return err } if resp.StatusCode != 200 { if resp.StatusCode == 403 && !strings.HasPrefix(endpoint, "https") { endpoint, err = httpsEndpoint(endpoint, body) if err != nil { return err } goto retry } return fmt.Errorf("got response %s %q", resp.Status, body) } err = json.Unmarshal(body, &v) if err != nil { return err } return nil } // PostV1 is a helper function to perform a V1 HTTP request // and parse our NSQ daemon's expected response format, with deadlines. func (c *Client) POSTV1(endpoint string, data url.Values, v interface{}) error { retry: var reqBody io.Reader if data != nil { js, err := json.Marshal(data) if err != nil { return fmt.Errorf("failed to marshal POST data to endpoint: %v", endpoint) } reqBody = bytes.NewBuffer(js) } req, err := http.NewRequest("POST", endpoint, reqBody) if err != nil { return err } req.Header.Add("Accept", "application/vnd.nsq; version=1.0") if reqBody != nil { req.Header.Add("Content-Type", "application/json") } resp, err := c.c.Do(req) if err != nil { return err } body, err := io.ReadAll(resp.Body) resp.Body.Close() if err != nil { return err } if resp.StatusCode != 200 { if resp.StatusCode == 403 && !strings.HasPrefix(endpoint, "https") { endpoint, err = httpsEndpoint(endpoint, body) if err != nil { return err } goto retry } return fmt.Errorf("got response %s %q", resp.Status, body) } if v != nil { return json.Unmarshal(body, &v) } return nil } func httpsEndpoint(endpoint string, body []byte) (string, error) { var forbiddenResp struct { HTTPSPort int `json:"https_port"` } err := json.Unmarshal(body, &forbiddenResp) if err != nil { return "", err } u, err := url.Parse(endpoint) if err != nil { return "", err } host, _, err := net.SplitHostPort(u.Host) if err != nil { return "", err } u.Scheme = "https" u.Host = net.JoinHostPort(host, strconv.Itoa(forbiddenResp.HTTPSPort)) return u.String(), nil } ================================================ FILE: internal/http_api/api_response.go ================================================ package http_api import ( "encoding/json" "fmt" "io" "net/http" "time" "github.com/julienschmidt/httprouter" "github.com/nsqio/nsq/internal/lg" ) type Decorator func(APIHandler) APIHandler type APIHandler func(http.ResponseWriter, *http.Request, httprouter.Params) (interface{}, error) type Err struct { Code int Text string } func (e Err) Error() string { return e.Text } func PlainText(f APIHandler) APIHandler { return func(w http.ResponseWriter, req *http.Request, ps httprouter.Params) (interface{}, error) { code := 200 data, err := f(w, req, ps) if err != nil { code = err.(Err).Code data = err.Error() } switch d := data.(type) { case string: w.WriteHeader(code) io.WriteString(w, d) case []byte: w.WriteHeader(code) w.Write(d) default: panic(fmt.Sprintf("unknown response type %T", data)) } return nil, nil } } func V1(f APIHandler) APIHandler { return func(w http.ResponseWriter, req *http.Request, ps httprouter.Params) (interface{}, error) { data, err := f(w, req, ps) if err != nil { RespondV1(w, err.(Err).Code, err) return nil, nil } RespondV1(w, 200, data) return nil, nil } } func RespondV1(w http.ResponseWriter, code int, data interface{}) { var response []byte var err error var isJSON bool if code == 200 { switch data := data.(type) { case string: response = []byte(data) case []byte: response = data case nil: response = []byte{} default: isJSON = true response, err = json.Marshal(data) if err != nil { code = 500 data = err } } } if code != 200 { isJSON = true response, _ = json.Marshal(struct { Message string `json:"message"` }{fmt.Sprintf("%s", data)}) } if isJSON { w.Header().Set("Content-Type", "application/json; charset=utf-8") } w.Header().Set("X-NSQ-Content-Type", "nsq; version=1.0") w.WriteHeader(code) w.Write(response) } func Decorate(f APIHandler, ds ...Decorator) httprouter.Handle { decorated := f for _, decorate := range ds { decorated = decorate(decorated) } return func(w http.ResponseWriter, req *http.Request, ps httprouter.Params) { decorated(w, req, ps) } } func Log(logf lg.AppLogFunc) Decorator { return func(f APIHandler) APIHandler { return func(w http.ResponseWriter, req *http.Request, ps httprouter.Params) (interface{}, error) { start := time.Now() response, err := f(w, req, ps) elapsed := time.Since(start) status := 200 if e, ok := err.(Err); ok { status = e.Code } logf(lg.INFO, "%d %s %s (%s) %s", status, req.Method, req.URL.RequestURI(), req.RemoteAddr, elapsed) return response, err } } } func LogPanicHandler(logf lg.AppLogFunc) func(w http.ResponseWriter, req *http.Request, p interface{}) { return func(w http.ResponseWriter, req *http.Request, p interface{}) { logf(lg.ERROR, "panic in HTTP handler - %s", p) Decorate(func(w http.ResponseWriter, req *http.Request, ps httprouter.Params) (interface{}, error) { return nil, Err{500, "INTERNAL_ERROR"} }, Log(logf), V1)(w, req, nil) } } func LogNotFoundHandler(logf lg.AppLogFunc) http.Handler { return http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) { Decorate(func(w http.ResponseWriter, req *http.Request, ps httprouter.Params) (interface{}, error) { return nil, Err{404, "NOT_FOUND"} }, Log(logf), V1)(w, req, nil) }) } func LogMethodNotAllowedHandler(logf lg.AppLogFunc) http.Handler { return http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) { Decorate(func(w http.ResponseWriter, req *http.Request, ps httprouter.Params) (interface{}, error) { return nil, Err{405, "METHOD_NOT_ALLOWED"} }, Log(logf), V1)(w, req, nil) }) } ================================================ FILE: internal/http_api/compress.go ================================================ // Copyright 2013 The Gorilla Authors. All rights reserved. // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. // copied from https://github.com/gorilla/handlers/blob/master/compress.go package http_api import ( "compress/flate" "compress/gzip" "io" "net/http" "strings" ) type compressResponseWriter struct { io.Writer http.ResponseWriter http.Hijacker } func (w *compressResponseWriter) Header() http.Header { return w.ResponseWriter.Header() } func (w *compressResponseWriter) WriteHeader(c int) { w.ResponseWriter.Header().Del("Content-Length") w.ResponseWriter.WriteHeader(c) } func (w *compressResponseWriter) Write(b []byte) (int, error) { h := w.ResponseWriter.Header() if h.Get("Content-Type") == "" { h.Set("Content-Type", http.DetectContentType(b)) } h.Del("Content-Length") return w.Writer.Write(b) } // CompressHandler gzip compresses HTTP responses for clients that support it // via the 'Accept-Encoding' header. func CompressHandler(h http.Handler) http.Handler { return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { L: for _, enc := range strings.Split(r.Header.Get("Accept-Encoding"), ",") { switch strings.TrimSpace(enc) { case "gzip": w.Header().Set("Content-Encoding", "gzip") w.Header().Add("Vary", "Accept-Encoding") gw := gzip.NewWriter(w) defer gw.Close() h, hok := w.(http.Hijacker) if !hok { /* w is not Hijacker... oh well... */ h = nil } w = &compressResponseWriter{ Writer: gw, ResponseWriter: w, Hijacker: h, } break L case "deflate": w.Header().Set("Content-Encoding", "deflate") w.Header().Add("Vary", "Accept-Encoding") fw, _ := flate.NewWriter(w, flate.DefaultCompression) defer fw.Close() h, hok := w.(http.Hijacker) if !hok { /* w is not Hijacker... oh well... */ h = nil } w = &compressResponseWriter{ Writer: fw, ResponseWriter: w, Hijacker: h, } break L } } h.ServeHTTP(w, r) }) } ================================================ FILE: internal/http_api/http_server.go ================================================ package http_api import ( "errors" "fmt" "log" "net" "net/http" "github.com/nsqio/nsq/internal/lg" ) type logWriter struct { logf lg.AppLogFunc } func (l logWriter) Write(p []byte) (int, error) { l.logf(lg.WARN, "%s", string(p)) return len(p), nil } func Serve(listener net.Listener, handler http.Handler, proto string, logf lg.AppLogFunc) error { logf(lg.INFO, "%s: listening on %s", proto, listener.Addr()) server := &http.Server{ Handler: handler, ErrorLog: log.New(logWriter{logf}, "", 0), } err := server.Serve(listener) // theres no direct way to detect this error because it is not exposed if err != nil && !errors.Is(err, net.ErrClosed) { return fmt.Errorf("http.Serve() error - %s", err) } logf(lg.INFO, "%s: closing %s", proto, listener.Addr()) return nil } ================================================ FILE: internal/http_api/req_params.go ================================================ package http_api import ( "errors" "io" "net/http" "net/url" ) type ReqParams struct { url.Values Body []byte } func NewReqParams(req *http.Request) (*ReqParams, error) { reqParams, err := url.ParseQuery(req.URL.RawQuery) if err != nil { return nil, err } data, err := io.ReadAll(req.Body) if err != nil { return nil, err } return &ReqParams{reqParams, data}, nil } func (r *ReqParams) Get(key string) (string, error) { v, ok := r.Values[key] if !ok { return "", errors.New("key not in query params") } return v[0], nil } func (r *ReqParams) GetAll(key string) ([]string, error) { v, ok := r.Values[key] if !ok { return nil, errors.New("key not in query params") } return v, nil } ================================================ FILE: internal/http_api/topic_channel_args.go ================================================ package http_api import ( "errors" "github.com/nsqio/nsq/internal/protocol" ) type getter interface { Get(key string) (string, error) } func GetTopicChannelArgs(rp getter) (string, string, error) { topicName, err := rp.Get("topic") if err != nil { return "", "", errors.New("MISSING_ARG_TOPIC") } if !protocol.IsValidTopicName(topicName) { return "", "", errors.New("INVALID_ARG_TOPIC") } channelName, err := rp.Get("channel") if err != nil { return "", "", errors.New("MISSING_ARG_CHANNEL") } if !protocol.IsValidChannelName(channelName) { return "", "", errors.New("INVALID_ARG_CHANNEL") } return topicName, channelName, nil } ================================================ FILE: internal/lg/lg.go ================================================ // Package lg provides leveled logging package lg import ( "fmt" "log" "os" "strings" ) const ( DEBUG = LogLevel(1) INFO = LogLevel(2) WARN = LogLevel(3) ERROR = LogLevel(4) FATAL = LogLevel(5) ) type AppLogFunc func(lvl LogLevel, f string, args ...interface{}) type Logger interface { Output(maxdepth int, s string) error } type NilLogger struct{} func (l NilLogger) Output(maxdepth int, s string) error { return nil } type LogLevel int func (l *LogLevel) Get() interface{} { return *l } func (l *LogLevel) Set(s string) error { lvl, err := ParseLogLevel(s) if err != nil { return err } *l = lvl return nil } func (l *LogLevel) String() string { switch *l { case DEBUG: return "DEBUG" case INFO: return "INFO" case WARN: return "WARNING" case ERROR: return "ERROR" case FATAL: return "FATAL" } return "invalid" } func ParseLogLevel(levelstr string) (LogLevel, error) { switch strings.ToLower(levelstr) { case "debug": return DEBUG, nil case "info": return INFO, nil case "warn": return WARN, nil case "error": return ERROR, nil case "fatal": return FATAL, nil } return 0, fmt.Errorf("invalid log level '%s' (debug, info, warn, error, fatal)", levelstr) } func Logf(logger Logger, cfgLevel LogLevel, msgLevel LogLevel, f string, args ...interface{}) { if cfgLevel > msgLevel { return } logger.Output(3, fmt.Sprintf(msgLevel.String()+": "+f, args...)) } func LogFatal(prefix string, f string, args ...interface{}) { logger := log.New(os.Stderr, prefix, log.Ldate|log.Ltime|log.Lmicroseconds) Logf(logger, FATAL, FATAL, f, args...) os.Exit(1) } ================================================ FILE: internal/lg/lg_test.go ================================================ package lg import ( "testing" "github.com/nsqio/nsq/internal/test" ) type mockLogger struct { Count int } func (l *mockLogger) Output(maxdepth int, s string) error { l.Count++ return nil } func TestLogging(t *testing.T) { logger := &mockLogger{} // Test only fatal get through logger.Count = 0 for i := 1; i <= 5; i++ { Logf(logger, FATAL, LogLevel(i), "Test") } test.Equal(t, 1, logger.Count) // Test only warnings or higher get through logger.Count = 0 for i := 1; i <= 5; i++ { Logf(logger, WARN, LogLevel(i), "Test") } test.Equal(t, 3, logger.Count) // Test everything gets through logger.Count = 0 for i := 1; i <= 5; i++ { Logf(logger, DEBUG, LogLevel(i), "Test") } test.Equal(t, 5, logger.Count) } ================================================ FILE: internal/pqueue/pqueue.go ================================================ package pqueue import ( "container/heap" ) type Item struct { Value interface{} Priority int64 Index int } // this is a priority queue as implemented by a min heap // ie. the 0th element is the *lowest* value type PriorityQueue []*Item func New(capacity int) PriorityQueue { return make(PriorityQueue, 0, capacity) } func (pq PriorityQueue) Len() int { return len(pq) } func (pq PriorityQueue) Less(i, j int) bool { return pq[i].Priority < pq[j].Priority } func (pq PriorityQueue) Swap(i, j int) { pq[i], pq[j] = pq[j], pq[i] pq[i].Index = i pq[j].Index = j } func (pq *PriorityQueue) Push(x interface{}) { n := len(*pq) c := cap(*pq) if n+1 > c { npq := make(PriorityQueue, n, c*2) copy(npq, *pq) *pq = npq } *pq = (*pq)[0 : n+1] item := x.(*Item) item.Index = n (*pq)[n] = item } func (pq *PriorityQueue) Pop() interface{} { n := len(*pq) c := cap(*pq) if n < (c/2) && c > 25 { npq := make(PriorityQueue, n, c/2) copy(npq, *pq) *pq = npq } item := (*pq)[n-1] item.Index = -1 *pq = (*pq)[0 : n-1] return item } func (pq *PriorityQueue) PeekAndShift(max int64) (*Item, int64) { if pq.Len() == 0 { return nil, 0 } item := (*pq)[0] if item.Priority > max { return nil, item.Priority - max } heap.Remove(pq, 0) return item, 0 } ================================================ FILE: internal/pqueue/pqueue_test.go ================================================ package pqueue import ( "container/heap" "math/rand" "path/filepath" "reflect" "runtime" "sort" "testing" ) func equal(t *testing.T, act, exp interface{}) { if !reflect.DeepEqual(exp, act) { _, file, line, _ := runtime.Caller(1) t.Logf("\033[31m%s:%d:\n\n\texp: %#v\n\n\tgot: %#v\033[39m\n\n", filepath.Base(file), line, exp, act) t.FailNow() } } func TestPriorityQueue(t *testing.T) { c := 100 pq := New(c) for i := 0; i < c+1; i++ { heap.Push(&pq, &Item{Value: i, Priority: int64(i)}) } equal(t, pq.Len(), c+1) equal(t, cap(pq), c*2) for i := 0; i < c+1; i++ { item := heap.Pop(&pq) equal(t, item.(*Item).Value.(int), i) } equal(t, cap(pq), c/4) } func TestUnsortedInsert(t *testing.T) { c := 100 pq := New(c) ints := make([]int, 0, c) for i := 0; i < c; i++ { v := rand.Int() ints = append(ints, v) heap.Push(&pq, &Item{Value: i, Priority: int64(v)}) } equal(t, pq.Len(), c) equal(t, cap(pq), c) sort.Ints(ints) for i := 0; i < c; i++ { item, _ := pq.PeekAndShift(int64(ints[len(ints)-1])) equal(t, item.Priority, int64(ints[i])) } } func TestRemove(t *testing.T) { c := 100 pq := New(c) for i := 0; i < c; i++ { v := rand.Int() heap.Push(&pq, &Item{Value: "test", Priority: int64(v)}) } for i := 0; i < 10; i++ { heap.Remove(&pq, rand.Intn((c-1)-i)) } lastPriority := heap.Pop(&pq).(*Item).Priority for i := 0; i < (c - 10 - 1); i++ { item := heap.Pop(&pq) equal(t, lastPriority < item.(*Item).Priority, true) lastPriority = item.(*Item).Priority } } ================================================ FILE: internal/protocol/byte_base10.go ================================================ package protocol import ( "errors" ) var errBase10 = errors.New("failed to convert to Base10") func ByteToBase10(b []byte) (n uint64, err error) { base := uint64(10) n = 0 for i := 0; i < len(b); i++ { var v byte d := b[i] switch { case '0' <= d && d <= '9': v = d - '0' default: n = 0 err = errBase10 return } n *= base n += uint64(v) } return n, err } ================================================ FILE: internal/protocol/byte_base10_test.go ================================================ package protocol import ( "testing" ) var result uint64 func BenchmarkByteToBase10Valid(b *testing.B) { bt := []byte{'3', '1', '4', '1', '5', '9', '2', '5'} var n uint64 for i := 0; i < b.N; i++ { n, _ = ByteToBase10(bt) } result = n } func BenchmarkByteToBase10Invalid(b *testing.B) { bt := []byte{'?', '1', '4', '1', '5', '9', '2', '5'} var n uint64 for i := 0; i < b.N; i++ { n, _ = ByteToBase10(bt) } result = n } ================================================ FILE: internal/protocol/errors.go ================================================ package protocol type ChildErr interface { Parent() error } // ClientErr provides a way for NSQ daemons to log a human reabable // error string and return a machine readable string to the client. // // see docs/protocol.md for error codes by command type ClientErr struct { ParentErr error Code string Desc string } // Error returns the machine readable form func (e *ClientErr) Error() string { return e.Code + " " + e.Desc } // Parent returns the parent error func (e *ClientErr) Parent() error { return e.ParentErr } // NewClientErr creates a ClientErr with the supplied human and machine readable strings func NewClientErr(parent error, code string, description string) *ClientErr { return &ClientErr{parent, code, description} } type FatalClientErr struct { ParentErr error Code string Desc string } // Error returns the machine readable form func (e *FatalClientErr) Error() string { return e.Code + " " + e.Desc } // Parent returns the parent error func (e *FatalClientErr) Parent() error { return e.ParentErr } // NewFatalClientErr creates a ClientErr with the supplied human and machine readable strings func NewFatalClientErr(parent error, code string, description string) *FatalClientErr { return &FatalClientErr{parent, code, description} } ================================================ FILE: internal/protocol/names.go ================================================ package protocol import ( "regexp" ) var validTopicChannelNameRegex = regexp.MustCompile(`^[.a-zA-Z0-9_-]+(#ephemeral)?$`) // IsValidTopicName checks a topic name for correctness func IsValidTopicName(name string) bool { return isValidName(name) } // IsValidChannelName checks a channel name for correctness func IsValidChannelName(name string) bool { return isValidName(name) } func isValidName(name string) bool { if len(name) > 64 || len(name) < 1 { return false } return validTopicChannelNameRegex.MatchString(name) } ================================================ FILE: internal/protocol/protocol.go ================================================ package protocol import ( "encoding/binary" "io" "net" ) type Client interface { Close() error } // Protocol describes the basic behavior of any protocol in the system type Protocol interface { NewClient(net.Conn) Client IOLoop(Client) error } // SendResponse is a server side utility function to prefix data with a length header // and write to the supplied Writer func SendResponse(w io.Writer, data []byte) (int, error) { err := binary.Write(w, binary.BigEndian, int32(len(data))) if err != nil { return 0, err } n, err := w.Write(data) if err != nil { return 0, err } return (n + 4), nil } // SendFramedResponse is a server side utility function to prefix data with a length header // and frame header and write to the supplied Writer func SendFramedResponse(w io.Writer, frameType int32, data []byte) (int, error) { beBuf := make([]byte, 4) size := uint32(len(data)) + 4 binary.BigEndian.PutUint32(beBuf, size) n, err := w.Write(beBuf) if err != nil { return n, err } binary.BigEndian.PutUint32(beBuf, uint32(frameType)) n, err = w.Write(beBuf) if err != nil { return n + 4, err } n, err = w.Write(data) return n + 8, err } ================================================ FILE: internal/protocol/tcp_server.go ================================================ package protocol import ( "errors" "fmt" "net" "runtime" "sync" "github.com/nsqio/nsq/internal/lg" ) type TCPHandler interface { Handle(net.Conn) } func TCPServer(listener net.Listener, handler TCPHandler, logf lg.AppLogFunc) error { logf(lg.INFO, "TCP: listening on %s", listener.Addr()) var wg sync.WaitGroup for { clientConn, err := listener.Accept() if err != nil { // net.Error.Temporary() is deprecated, but is valid for accept // this is a hack to avoid a staticcheck error if te, ok := err.(interface{ Temporary() bool }); ok && te.Temporary() { logf(lg.WARN, "temporary Accept() failure - %s", err) runtime.Gosched() continue } // theres no direct way to detect this error because it is not exposed if !errors.Is(err, net.ErrClosed) { return fmt.Errorf("listener.Accept() error - %s", err) } break } wg.Add(1) go func() { handler.Handle(clientConn) wg.Done() }() } // wait to return until all handler goroutines complete wg.Wait() logf(lg.INFO, "TCP: closing %s", listener.Addr()) return nil } ================================================ FILE: internal/quantile/aggregate.go ================================================ package quantile import ( "encoding/json" "math" "sort" ) type E2eProcessingLatencyAggregate struct { Count int `json:"count"` Percentiles []map[string]float64 `json:"percentiles"` Topic string `json:"topic"` Channel string `json:"channel"` Addr string `json:"host"` } func (e *E2eProcessingLatencyAggregate) UnmarshalJSON(b []byte) error { var resp struct { Count int `json:"count"` Percentiles []map[string]float64 `json:"percentiles"` Topic string `json:"topic"` Channel string `json:"channel"` Addr string `json:"host"` } err := json.Unmarshal(b, &resp) if err != nil { return err } for _, p := range resp.Percentiles { p["min"] = p["value"] p["max"] = p["value"] p["average"] = p["value"] p["count"] = float64(resp.Count) } e.Count = resp.Count e.Percentiles = resp.Percentiles e.Topic = resp.Topic e.Channel = resp.Channel e.Addr = resp.Addr return nil } func (e *E2eProcessingLatencyAggregate) Len() int { return len(e.Percentiles) } func (e *E2eProcessingLatencyAggregate) Swap(i, j int) { e.Percentiles[i], e.Percentiles[j] = e.Percentiles[j], e.Percentiles[i] } func (e *E2eProcessingLatencyAggregate) Less(i, j int) bool { return e.Percentiles[i]["percentile"] > e.Percentiles[j]["percentile"] } // Add merges e2 into e by averaging the percentiles func (e *E2eProcessingLatencyAggregate) Add(e2 *E2eProcessingLatencyAggregate) { e.Addr = "*" p := e.Percentiles e.Count += e2.Count for _, value := range e2.Percentiles { i := -1 for j, v := range p { if value["quantile"] == v["quantile"] { i = j break } } if i == -1 { i = len(p) e.Percentiles = append(p, make(map[string]float64)) p = e.Percentiles p[i]["quantile"] = value["quantile"] } p[i]["max"] = math.Max(value["max"], p[i]["max"]) p[i]["min"] = math.Min(value["max"], p[i]["max"]) p[i]["count"] += value["count"] if p[i]["count"] == 0 { p[i]["average"] = 0 continue } delta := value["average"] - p[i]["average"] R := delta * value["count"] / p[i]["count"] p[i]["average"] = p[i]["average"] + R } sort.Sort(e) } ================================================ FILE: internal/quantile/quantile.go ================================================ package quantile import ( "strings" "sync" "time" "github.com/bmizerany/perks/quantile" "github.com/nsqio/nsq/internal/stringy" ) type Result struct { Count int `json:"count"` Percentiles []map[string]float64 `json:"percentiles"` } func (r *Result) String() string { var s []string for _, item := range r.Percentiles { s = append(s, stringy.NanoSecondToHuman(item["value"])) } return strings.Join(s, ", ") } type Quantile struct { sync.Mutex streams [2]quantile.Stream currentIndex uint8 lastMoveWindow time.Time currentStream *quantile.Stream Percentiles []float64 MoveWindowTime time.Duration } func New(WindowTime time.Duration, Percentiles []float64) *Quantile { q := Quantile{ currentIndex: 0, lastMoveWindow: time.Now(), MoveWindowTime: WindowTime / 2, Percentiles: Percentiles, } for i := 0; i < 2; i++ { q.streams[i] = *quantile.NewTargeted(Percentiles...) } q.currentStream = &q.streams[0] return &q } func (q *Quantile) Result() *Result { if q == nil { return &Result{} } queryHandler := q.QueryHandler() result := Result{ Count: queryHandler.Count(), Percentiles: make([]map[string]float64, len(q.Percentiles)), } for i, p := range q.Percentiles { value := queryHandler.Query(p) result.Percentiles[i] = map[string]float64{"quantile": p, "value": value} } return &result } func (q *Quantile) Insert(msgStartTime int64) { q.Lock() now := time.Now() for q.IsDataStale(now) { q.moveWindow() } q.currentStream.Insert(float64(now.UnixNano() - msgStartTime)) q.Unlock() } func (q *Quantile) QueryHandler() *quantile.Stream { q.Lock() now := time.Now() for q.IsDataStale(now) { q.moveWindow() } merged := quantile.NewTargeted(q.Percentiles...) merged.Merge(q.streams[0].Samples()) merged.Merge(q.streams[1].Samples()) q.Unlock() return merged } func (q *Quantile) IsDataStale(now time.Time) bool { return now.After(q.lastMoveWindow.Add(q.MoveWindowTime)) } func (q *Quantile) Merge(them *Quantile) { q.Lock() them.Lock() iUs := q.currentIndex iThem := them.currentIndex q.streams[iUs].Merge(them.streams[iThem].Samples()) iUs ^= 0x1 iThem ^= 0x1 q.streams[iUs].Merge(them.streams[iThem].Samples()) if q.lastMoveWindow.Before(them.lastMoveWindow) { q.lastMoveWindow = them.lastMoveWindow } q.Unlock() them.Unlock() } func (q *Quantile) moveWindow() { q.currentIndex ^= 0x1 q.currentStream = &q.streams[q.currentIndex] q.lastMoveWindow = q.lastMoveWindow.Add(q.MoveWindowTime) q.currentStream.Reset() } ================================================ FILE: internal/statsd/client.go ================================================ package statsd import ( "fmt" "io" ) type Client struct { w io.Writer prefix string } func NewClient(w io.Writer, prefix string) *Client { return &Client{ w: w, prefix: prefix, } } func (c *Client) Incr(stat string, count int64) error { return c.send(stat, "%d|c", count) } func (c *Client) Decr(stat string, count int64) error { return c.send(stat, "%d|c", -count) } func (c *Client) Timing(stat string, delta int64) error { return c.send(stat, "%d|ms", delta) } func (c *Client) Gauge(stat string, value int64) error { return c.send(stat, "%d|g", value) } func (c *Client) send(stat string, format string, value int64) error { format = fmt.Sprintf("%s%s:%s\n", c.prefix, stat, format) _, err := fmt.Fprintf(c.w, format, value) return err } ================================================ FILE: internal/statsd/host.go ================================================ package statsd import ( "strings" ) func HostKey(h string) string { return strings.Replace(strings.Replace(h, ".", "_", -1), ":", "_", -1) } ================================================ FILE: internal/stringy/slice.go ================================================ package stringy func Add(s []string, a string) []string { for _, existing := range s { if a == existing { return s } } return append(s, a) } func Union(s []string, a []string) []string { for _, entry := range a { found := false for _, existing := range s { if entry == existing { found = true break } } if !found { s = append(s, entry) } } return s } func Uniq(s []string) (r []string) { outerLoop: for _, entry := range s { for _, existing := range r { if existing == entry { continue outerLoop } } r = append(r, entry) } return } ================================================ FILE: internal/stringy/slice_test.go ================================================ package stringy_test import ( "testing" "github.com/nsqio/nsq/internal/stringy" ) func BenchmarkUniq(b *testing.B) { values := []string{"a", "a", "a", "a", "a", "a", "a", "a", "a", "a", "a", "a", "a", "a", "a", "a", "a", "a", "a", "a", "a", "a", "a", "a", "a", "a", "a", "a", "a", "a", "b"} for i := 0; i < b.N; i++ { values = stringy.Uniq(values) if len(values) != 2 { b.Fatal("values len is incorrect") } } } func TestUniq(t *testing.T) { values := []string{"a", "a", "a", "b", "b", "b", "c", "c", "c"} values = stringy.Uniq(values) if len(values) != 3 { t.Fatal("values len is incorrect") } } ================================================ FILE: internal/stringy/template.go ================================================ package stringy import ( "fmt" ) func NanoSecondToHuman(v float64) string { var suffix string switch { case v > 1000000000: v /= 1000000000 suffix = "s" case v > 1000000: v /= 1000000 suffix = "ms" case v > 1000: v /= 1000 suffix = "us" default: suffix = "ns" } return fmt.Sprintf("%0.1f%s", v, suffix) } ================================================ FILE: internal/test/assertions.go ================================================ package test import ( "path/filepath" "reflect" "runtime" "testing" ) func Equal(t *testing.T, expected, actual interface{}) { if !reflect.DeepEqual(expected, actual) { _, file, line, _ := runtime.Caller(1) t.Logf("\033[31m%s:%d:\n\n\t %#v (expected)\n\n\t!= %#v (actual)\033[39m\n\n", filepath.Base(file), line, expected, actual) t.FailNow() } } func NotEqual(t *testing.T, expected, actual interface{}) { if reflect.DeepEqual(expected, actual) { _, file, line, _ := runtime.Caller(1) t.Logf("\033[31m%s:%d:\n\n\tnexp: %#v\n\n\tgot: %#v\033[39m\n\n", filepath.Base(file), line, expected, actual) t.FailNow() } } func Nil(t *testing.T, object interface{}) { if !isNil(object) { _, file, line, _ := runtime.Caller(1) t.Logf("\033[31m%s:%d:\n\n\t (expected)\n\n\t!= %#v (actual)\033[39m\n\n", filepath.Base(file), line, object) t.FailNow() } } func NotNil(t *testing.T, object interface{}) { if isNil(object) { _, file, line, _ := runtime.Caller(1) t.Logf("\033[31m%s:%d:\n\n\tExpected value not to be \033[39m\n\n", filepath.Base(file), line) t.FailNow() } } func isNil(object interface{}) bool { if object == nil { return true } value := reflect.ValueOf(object) kind := value.Kind() if kind >= reflect.Chan && kind <= reflect.Slice && value.IsNil() { return true } return false } ================================================ FILE: internal/test/fakes.go ================================================ package test import ( "net" "time" ) type FakeNetConn struct { ReadFunc func([]byte) (int, error) WriteFunc func([]byte) (int, error) CloseFunc func() error LocalAddrFunc func() net.Addr RemoteAddrFunc func() net.Addr SetDeadlineFunc func(time.Time) error SetReadDeadlineFunc func(time.Time) error SetWriteDeadlineFunc func(time.Time) error } func (f FakeNetConn) Read(b []byte) (int, error) { return f.ReadFunc(b) } func (f FakeNetConn) Write(b []byte) (int, error) { return f.WriteFunc(b) } func (f FakeNetConn) Close() error { return f.CloseFunc() } func (f FakeNetConn) LocalAddr() net.Addr { return f.LocalAddrFunc() } func (f FakeNetConn) RemoteAddr() net.Addr { return f.RemoteAddrFunc() } func (f FakeNetConn) SetDeadline(t time.Time) error { return f.SetDeadlineFunc(t) } func (f FakeNetConn) SetReadDeadline(t time.Time) error { return f.SetReadDeadlineFunc(t) } func (f FakeNetConn) SetWriteDeadline(t time.Time) error { return f.SetWriteDeadlineFunc(t) } type fakeNetAddr struct{} func (fakeNetAddr) Network() string { return "" } func (fakeNetAddr) String() string { return "" } func NewFakeNetConn() FakeNetConn { netAddr := fakeNetAddr{} return FakeNetConn{ ReadFunc: func(b []byte) (int, error) { return 0, nil }, WriteFunc: func(b []byte) (int, error) { return len(b), nil }, CloseFunc: func() error { return nil }, LocalAddrFunc: func() net.Addr { return netAddr }, RemoteAddrFunc: func() net.Addr { return netAddr }, SetDeadlineFunc: func(time.Time) error { return nil }, SetWriteDeadlineFunc: func(time.Time) error { return nil }, SetReadDeadlineFunc: func(time.Time) error { return nil }, } } ================================================ FILE: internal/test/logger.go ================================================ package test type Logger interface { Output(maxdepth int, s string) error } type tbLog interface { Log(...interface{}) } type testLogger struct { tbLog } func (tl *testLogger) Output(maxdepth int, s string) error { tl.Log(s) return nil } func NewTestLogger(tbl tbLog) Logger { return &testLogger{tbl} } ================================================ FILE: internal/util/rand.go ================================================ package util import ( "math/rand" ) func UniqRands(quantity int, maxval int) []int { if maxval < quantity { quantity = maxval } intSlice := make([]int, maxval) for i := 0; i < maxval; i++ { intSlice[i] = i } for i := 0; i < quantity; i++ { j := rand.Int()%maxval + i // swap intSlice[i], intSlice[j] = intSlice[j], intSlice[i] maxval-- } return intSlice[0:quantity] } ================================================ FILE: internal/util/unix_socket.go ================================================ package util import ( "net" ) func TypeOfAddr(addr string) string { if _, _, err := net.SplitHostPort(addr); err == nil { return "tcp" } return "unix" } ================================================ FILE: internal/util/util_test.go ================================================ package util import ( "testing" "github.com/nsqio/nsq/internal/test" ) func BenchmarkUniqRands5of5(b *testing.B) { for i := 0; i < b.N; i++ { UniqRands(5, 5) } } func BenchmarkUniqRands20of20(b *testing.B) { for i := 0; i < b.N; i++ { UniqRands(20, 20) } } func BenchmarkUniqRands20of50(b *testing.B) { for i := 0; i < b.N; i++ { UniqRands(20, 50) } } func TestUniqRands(t *testing.T) { var x []int x = UniqRands(3, 10) test.Equal(t, 3, len(x)) x = UniqRands(10, 5) test.Equal(t, 5, len(x)) x = UniqRands(10, 20) test.Equal(t, 10, len(x)) } func TestTypeOfAddr(t *testing.T) { var x string x = TypeOfAddr("127.0.0.1:80") test.Equal(t, "tcp", x) x = TypeOfAddr("test:80") test.Equal(t, "tcp", x) x = TypeOfAddr("/var/run/nsqd.sock") test.Equal(t, "unix", x) x = TypeOfAddr("[::1%lo0]:80") test.Equal(t, "tcp", x) } ================================================ FILE: internal/util/wait_group_wrapper.go ================================================ package util import ( "sync" ) type WaitGroupWrapper struct { sync.WaitGroup } func (w *WaitGroupWrapper) Wrap(cb func()) { w.Add(1) go func() { cb() w.Done() }() } ================================================ FILE: internal/version/binary.go ================================================ package version import ( "fmt" "runtime" ) const Binary = "1.3.0" func String(app string) string { return fmt.Sprintf("%s v%s (built w/%s)", app, Binary, runtime.Version()) } ================================================ FILE: internal/writers/boundary_buffered_writer.go ================================================ package writers import ( "bufio" "io" ) type BoundaryBufferedWriter struct { bw *bufio.Writer } func NewBoundaryBufferedWriter(w io.Writer, size int) *BoundaryBufferedWriter { return &BoundaryBufferedWriter{ bw: bufio.NewWriterSize(w, size), } } func (b *BoundaryBufferedWriter) Write(p []byte) (int, error) { if len(p) > b.bw.Available() { err := b.bw.Flush() if err != nil { return 0, err } } return b.bw.Write(p) } func (b *BoundaryBufferedWriter) Flush() error { return b.bw.Flush() } ================================================ FILE: internal/writers/spread_writer.go ================================================ package writers import ( "io" "time" ) type SpreadWriter struct { w io.Writer interval time.Duration buf [][]byte exitCh chan int } func NewSpreadWriter(w io.Writer, interval time.Duration, exitCh chan int) *SpreadWriter { return &SpreadWriter{ w: w, interval: interval, buf: make([][]byte, 0), exitCh: exitCh, } } func (s *SpreadWriter) Write(p []byte) (int, error) { b := make([]byte, len(p)) copy(b, p) s.buf = append(s.buf, b) return len(p), nil } func (s *SpreadWriter) Flush() { if len(s.buf) == 0 { // nothing to write, just wait select { case <-time.After(s.interval): case <-s.exitCh: } return } sleep := s.interval / time.Duration(len(s.buf)) ticker := time.NewTicker(sleep) for _, b := range s.buf { s.w.Write(b) select { case <-ticker.C: case <-s.exitCh: // skip sleeps finish writes } } ticker.Stop() s.buf = s.buf[:0] } ================================================ FILE: nsqadmin/.eslintrc ================================================ { "env": { "browser": true, "es2020": true }, // Rule docs: http://eslint.org/docs/rules/ "rules": { "block-scoped-var": [1], "brace-style": [1, "1tbs", {"allowSingleLine": true}], "camelcase": [1], "comma-spacing": [1], "comma-style": [1], "computed-property-spacing": [1, "never"], "consistent-return": [1], "consistent-this": [1, "self"], "curly": [2], "dot-notation": [0], "eol-last": [1], "eqeqeq": [1], "indent": [1, 4], "key-spacing": [1], "max-len": [1, 100], "max-nested-callbacks": [2, 3], // ???? "new-cap": [1], "new-parens": [1], "no-caller": [2], "no-console": [0], "no-eval": [2], "no-extend-native": [2], "no-extra-bind": [1], "no-floating-decimal": [1], "no-iterator": [1], "no-lone-blocks": [1], "no-lonely-if": [1], "no-mixed-requires": [0], "no-mixed-spaces-and-tabs": [1], "no-multi-spaces": [1], "no-multi-str": [1], "no-multiple-empty-lines": [2, {"max": 2}], "no-native-reassign": [1], "no-new": [0], "no-redeclare": [1], "no-shadow": [1], "no-spaced-func": [1], "no-throw-literal": [1], "no-trailing-spaces": [1], "no-undef": [1], "no-underscore-dangle": [0], "no-unneeded-ternary": [1], "no-unused-vars": [1], "no-use-before-define": [1, "nofunc"], "no-with": [2], "one-var": [1, "never"], "quotes": [1, "single"], "radix": [1], "semi": [1, "always"], "semi-spacing": [1], "keyword-spacing": [1, {"before": true, "after": true}], "space-before-blocks": [1, "always"], "space-before-function-paren": [1, {"anonymous": "never", "named": "never", "asyncArrow": "always"}], "space-in-parens": [1, "never"], "space-infix-ops": [1], "space-unary-ops": [1], "strict": [0], "wrap-iife": [1] }, "globals": { "BASE_PATH": true, "GRAPHITE_URL": true, "GRAPH_ENABLED": true, "IS_ADMIN": true, "NSQLOOKUPD": true, "STATSD_COUNTER_FORMAT": true, "STATSD_GAUGE_FORMAT": true, "STATSD_INTERVAL": true, "STATSD_PREFIX": true, "USER_AGENT": true, "VERSION": true, "module": true, "require": true } } ================================================ FILE: nsqadmin/README.md ================================================ ## nsqadmin `nsqadmin` is a Web UI to view aggregated cluster stats in realtime and perform various administrative tasks. Read the [docs](https://nsq.io/components/nsqadmin.html) ## Local Development ### Dependencies 1. Install NodeJS 16.x (includes `npm`) ### Live Reload Workflow 1. `$ npm install` 2. `$ ./gulp --series clean watch` 3. `$ cd .. && make && ./build/nsqadmin --dev-static-dir=nsqadmin/static/build --lookupd-http-address=<...>` 4. make changes to static assets (repeat step 3 only if you make changes to any Go code) ### Build 1. `$ ./gulp --series clean build` ================================================ FILE: nsqadmin/gulp ================================================ #!/bin/sh exec ./node_modules/.bin/gulp "$@" ================================================ FILE: nsqadmin/gulpfile.js ================================================ var browserify = require('browserify'); var clean = require('gulp-clean'); var gulp = require('gulp'); var notify = require('gulp-notify'); var path = require('path'); var sass = require('gulp-dart-sass'); var source = require('vinyl-source-stream'); var taskListing = require('gulp-task-listing'); var uglify = require('gulp-uglify'); var sourcemaps = require('gulp-sourcemaps'); var buffer = require('vinyl-buffer'); var ROOT = 'static'; var VENDOR_CONFIG = { 'src': [ 'backbone', 'jquery', 'underscore', 'bootbox', ], 'target': 'vendor.js', 'targetDir': './static/build/' }; function excludeVendor(b) { VENDOR_CONFIG.src.forEach(function(vendorLib) { b.exclude(vendorLib); }); } function bytesToKB(bytes) { return Math.floor(+bytes/1024); } function logBundle(filename, watching) { return function (err, buf) { if (err) { console.error(err.toString()); if (!watching) { process.exit(1); } } if (!watching) { console.log(filename + ' ' + bytesToKB(buf.length) + ' KB written'); } } } function sassTask(root, inputFile) { return function sassing() { var onError = function(err) { notify({'title': 'Sass Compile Error'}).write(err); }; return gulp.src(path.join(root, 'css', inputFile)) .pipe(sass({ 'sourceComments': 'map', }).on('error', onError)) .pipe(gulp.dest(path.join(root, 'build/'))); }; } function browserifyTask(root, inputFile) { return function browserifying() { var onError = function() { var args = Array.prototype.slice.call(arguments); notify.onError({ 'title': 'JS Compile Error', 'message': '<%= error.message %>' }).apply(this, args); // Keep gulp from hanging on this task this.emit('end'); }; // Browserify needs a node module to import as its arg, so we need to // force the leading "./" to be included. var b = browserify({ entries: './' + path.join(root, 'js', inputFile), debug: true }) excludeVendor(b); return b.bundle() .pipe(source(inputFile)) .pipe(buffer()) .pipe(sourcemaps.init({'loadMaps': true, 'debug': true})) // Add transformation tasks to the pipeline here. .pipe(uglify()) .on('error', onError) .pipe(sourcemaps.write('./')) .pipe(gulp.dest(path.join(root, 'build/'))); }; } function watchTask(root) { return function watching() { gulp.watch(path.join(root, 'sass/**/*.scss'), gulp.series('sass')); gulp.watch([ path.join(root, 'js/**/*.js'), path.join(root, 'js/**/*.hbs') ], gulp.series('browserify')); gulp.watch([ path.join(root, 'html/**'), path.join(root, 'fonts/**') ], gulp.series('sync-static-assets')) }; } function cleanTask() { var paths = Array.prototype.slice.apply(arguments); return function cleaning() { return gulp.src(paths, {allowEmpty: true}).pipe(clean()); }; } gulp.task('vendor-build-js', function() { var onError = function() { var args = Array.prototype.slice.call(arguments); notify.onError({ 'title': 'JS Compile Error', 'message': '<%= error.message %>' }).apply(this, args); // Keep gulp from hanging on this task this.emit('end'); }; var b = browserify() .require(VENDOR_CONFIG.src); return b.bundle(logBundle(VENDOR_CONFIG.target)) .pipe(source(VENDOR_CONFIG.target)) .pipe(buffer()) // Add transformation tasks to the pipeline here. .pipe(uglify()) .on('error', onError) .pipe(gulp.dest(VENDOR_CONFIG.targetDir)); }); gulp.task('help', taskListing); gulp.task('sync-static-assets', function() { return gulp.src([ path.join(ROOT, 'html/**'), path.join(ROOT, 'fonts/**'), path.join(ROOT, 'img/**') ]).pipe(gulp.dest(path.join(ROOT, 'build'))); }); gulp.task('sass', sassTask(ROOT, '*.*css')); gulp.task('browserify', browserifyTask(ROOT, 'main.js')); gulp.task('build', gulp.parallel('sass', 'browserify', 'sync-static-assets', 'vendor-build-js')); gulp.task('watch', gulp.series('build', watchTask(ROOT))); gulp.task('clean', gulp.series(cleanTask(path.join(ROOT, 'build')))); gulp.task('default', gulp.series('help')); ================================================ FILE: nsqadmin/http.go ================================================ package nsqadmin import ( "encoding/json" "fmt" "html/template" "io" "mime" "net" "net/http" "net/http/httputil" "net/url" "os" "path" "reflect" "sort" "strings" "time" "github.com/julienschmidt/httprouter" "github.com/nsqio/nsq/internal/clusterinfo" "github.com/nsqio/nsq/internal/http_api" "github.com/nsqio/nsq/internal/lg" "github.com/nsqio/nsq/internal/protocol" "github.com/nsqio/nsq/internal/version" ) func maybeWarnMsg(msgs []string) string { if len(msgs) > 0 { return "WARNING: " + strings.Join(msgs, "; ") } return "" } // this is similar to httputil.NewSingleHostReverseProxy except it passes along basic auth func NewSingleHostReverseProxy(target *url.URL, connectTimeout time.Duration, requestTimeout time.Duration) *httputil.ReverseProxy { director := func(req *http.Request) { req.URL.Scheme = target.Scheme req.URL.Host = target.Host if target.User != nil { passwd, _ := target.User.Password() req.SetBasicAuth(target.User.Username(), passwd) } } return &httputil.ReverseProxy{ Director: director, Transport: http_api.NewDeadlineTransport(connectTimeout, requestTimeout), } } type httpServer struct { nsqadmin *NSQAdmin router http.Handler client *http_api.Client ci *clusterinfo.ClusterInfo basePath string devStaticDir string } func NewHTTPServer(nsqadmin *NSQAdmin) *httpServer { log := http_api.Log(nsqadmin.logf) client := http_api.NewClient(nsqadmin.httpClientTLSConfig, nsqadmin.getOpts().HTTPClientConnectTimeout, nsqadmin.getOpts().HTTPClientRequestTimeout) router := httprouter.New() router.HandleMethodNotAllowed = true router.PanicHandler = http_api.LogPanicHandler(nsqadmin.logf) router.NotFound = http_api.LogNotFoundHandler(nsqadmin.logf) router.MethodNotAllowed = http_api.LogMethodNotAllowedHandler(nsqadmin.logf) s := &httpServer{ nsqadmin: nsqadmin, router: router, client: client, ci: clusterinfo.New(nsqadmin.logf, client), basePath: nsqadmin.getOpts().BasePath, devStaticDir: nsqadmin.getOpts().DevStaticDir, } bp := func(p string) string { return path.Join(s.basePath, p) } router.Handle("GET", bp("/"), http_api.Decorate(s.indexHandler, log)) router.Handle("GET", bp("/ping"), http_api.Decorate(s.pingHandler, log, http_api.PlainText)) router.Handle("GET", bp("/topics"), http_api.Decorate(s.indexHandler, log)) router.Handle("GET", bp("/topics/:topic"), http_api.Decorate(s.indexHandler, log)) router.Handle("GET", bp("/topics/:topic/:channel"), http_api.Decorate(s.indexHandler, log)) router.Handle("GET", bp("/nodes"), http_api.Decorate(s.indexHandler, log)) router.Handle("GET", bp("/nodes/:node"), http_api.Decorate(s.indexHandler, log)) router.Handle("GET", bp("/counter"), http_api.Decorate(s.indexHandler, log)) router.Handle("GET", bp("/lookup"), http_api.Decorate(s.indexHandler, log)) router.Handle("GET", bp("/static/:asset"), http_api.Decorate(s.staticAssetHandler, log, http_api.PlainText)) router.Handle("GET", bp("/fonts/:asset"), http_api.Decorate(s.staticAssetHandler, log, http_api.PlainText)) if s.nsqadmin.getOpts().ProxyGraphite { proxy := NewSingleHostReverseProxy(nsqadmin.graphiteURL, nsqadmin.getOpts().HTTPClientConnectTimeout, nsqadmin.getOpts().HTTPClientRequestTimeout) router.Handler("GET", bp("/render"), proxy) } // v1 endpoints router.Handle("GET", bp("/api/topics"), http_api.Decorate(s.topicsHandler, log, http_api.V1)) router.Handle("GET", bp("/api/topics/:topic"), http_api.Decorate(s.topicHandler, log, http_api.V1)) router.Handle("GET", bp("/api/topics/:topic/:channel"), http_api.Decorate(s.channelHandler, log, http_api.V1)) router.Handle("GET", bp("/api/nodes"), http_api.Decorate(s.nodesHandler, log, http_api.V1)) router.Handle("GET", bp("/api/nodes/:node"), http_api.Decorate(s.nodeHandler, log, http_api.V1)) router.Handle("POST", bp("/api/topics"), http_api.Decorate(s.createTopicChannelHandler, log, http_api.V1)) router.Handle("POST", bp("/api/topics/:topic"), http_api.Decorate(s.topicActionHandler, log, http_api.V1)) router.Handle("POST", bp("/api/topics/:topic/:channel"), http_api.Decorate(s.channelActionHandler, log, http_api.V1)) router.Handle("DELETE", bp("/api/nodes/:node"), http_api.Decorate(s.tombstoneNodeForTopicHandler, log, http_api.V1)) router.Handle("DELETE", bp("/api/topics/:topic"), http_api.Decorate(s.deleteTopicHandler, log, http_api.V1)) router.Handle("DELETE", bp("/api/topics/:topic/:channel"), http_api.Decorate(s.deleteChannelHandler, log, http_api.V1)) router.Handle("GET", bp("/api/counter"), http_api.Decorate(s.counterHandler, log, http_api.V1)) router.Handle("GET", bp("/api/graphite"), http_api.Decorate(s.graphiteHandler, log, http_api.V1)) router.Handle("GET", bp("/config/:opt"), http_api.Decorate(s.doConfig, log, http_api.V1)) router.Handle("PUT", bp("/config/:opt"), http_api.Decorate(s.doConfig, log, http_api.V1)) return s } func (s *httpServer) ServeHTTP(w http.ResponseWriter, req *http.Request) { s.router.ServeHTTP(w, req) } func (s *httpServer) pingHandler(w http.ResponseWriter, req *http.Request, ps httprouter.Params) (interface{}, error) { return "OK", nil } func (s *httpServer) indexHandler(w http.ResponseWriter, req *http.Request, ps httprouter.Params) (interface{}, error) { asset, _ := staticAsset("index.html") t, _ := template.New("index").Funcs(template.FuncMap{ "basePath": func(p string) string { return path.Join(s.basePath, p) }, }).Parse(string(asset)) w.Header().Set("Content-Type", "text/html") t.Execute(w, struct { Version string ProxyGraphite bool GraphEnabled bool GraphiteURL string StatsdInterval int StatsdCounterFormat string StatsdGaugeFormat string StatsdPrefix string NSQLookupd []string IsAdmin bool }{ Version: version.Binary, ProxyGraphite: s.nsqadmin.getOpts().ProxyGraphite, GraphEnabled: s.nsqadmin.getOpts().GraphiteURL != "", GraphiteURL: s.nsqadmin.getOpts().GraphiteURL, StatsdInterval: int(s.nsqadmin.getOpts().StatsdInterval / time.Second), StatsdCounterFormat: s.nsqadmin.getOpts().StatsdCounterFormat, StatsdGaugeFormat: s.nsqadmin.getOpts().StatsdGaugeFormat, StatsdPrefix: s.nsqadmin.getOpts().StatsdPrefix, NSQLookupd: s.nsqadmin.getOpts().NSQLookupdHTTPAddresses, IsAdmin: s.isAuthorizedAdminRequest(req), }) return nil, nil } func (s *httpServer) staticAssetHandler(w http.ResponseWriter, req *http.Request, ps httprouter.Params) (interface{}, error) { assetName := ps.ByName("asset") var ( asset []byte err error ) if s.devStaticDir != "" { s.nsqadmin.logf(LOG_DEBUG, "using dev dir %q for static asset %q", s.devStaticDir, assetName) fsPath := path.Join(s.devStaticDir, assetName) asset, err = os.ReadFile(fsPath) } else { asset, err = staticAsset(assetName) } if err != nil { return nil, http_api.Err{404, "NOT_FOUND"} } ext := path.Ext(assetName) ct := mime.TypeByExtension(ext) if ct == "" { switch ext { case ".map": ct = "application/json" case ".svg": ct = "image/svg+xml" case ".woff": ct = "application/font-woff" case ".ttf": ct = "application/font-sfnt" case ".eot": ct = "application/vnd.ms-fontobject" case ".woff2": ct = "application/font-woff2" } } if ct != "" { w.Header().Set("Content-Type", ct) } return string(asset), nil } func (s *httpServer) topicsHandler(w http.ResponseWriter, req *http.Request, ps httprouter.Params) (interface{}, error) { var messages []string reqParams, err := http_api.NewReqParams(req) if err != nil { return nil, http_api.Err{400, err.Error()} } var topics []string if len(s.nsqadmin.getOpts().NSQLookupdHTTPAddresses) != 0 { topics, err = s.ci.GetLookupdTopics(s.nsqadmin.getOpts().NSQLookupdHTTPAddresses) } else { topics, err = s.ci.GetNSQDTopics(s.nsqadmin.getOpts().NSQDHTTPAddresses) } if err != nil { pe, ok := err.(clusterinfo.PartialErr) if !ok { s.nsqadmin.logf(LOG_ERROR, "failed to get topics - %s", err) return nil, http_api.Err{502, fmt.Sprintf("UPSTREAM_ERROR: %s", err)} } s.nsqadmin.logf(LOG_WARN, "%s", err) messages = append(messages, pe.Error()) } inactive, _ := reqParams.Get("inactive") if inactive == "true" { topicChannelMap := make(map[string][]string) if len(s.nsqadmin.getOpts().NSQLookupdHTTPAddresses) == 0 { goto respond } for _, topicName := range topics { producers, _ := s.ci.GetLookupdTopicProducers( topicName, s.nsqadmin.getOpts().NSQLookupdHTTPAddresses) if len(producers) == 0 { topicChannels, _ := s.ci.GetLookupdTopicChannels( topicName, s.nsqadmin.getOpts().NSQLookupdHTTPAddresses) topicChannelMap[topicName] = topicChannels } } respond: return struct { Topics map[string][]string `json:"topics"` Message string `json:"message"` }{topicChannelMap, maybeWarnMsg(messages)}, nil } return struct { Topics []string `json:"topics"` Message string `json:"message"` }{topics, maybeWarnMsg(messages)}, nil } func (s *httpServer) topicHandler(w http.ResponseWriter, req *http.Request, ps httprouter.Params) (interface{}, error) { var messages []string topicName := ps.ByName("topic") producers, err := s.ci.GetTopicProducers(topicName, s.nsqadmin.getOpts().NSQLookupdHTTPAddresses, s.nsqadmin.getOpts().NSQDHTTPAddresses) if err != nil { pe, ok := err.(clusterinfo.PartialErr) if !ok { s.nsqadmin.logf(LOG_ERROR, "failed to get topic producers - %s", err) return nil, http_api.Err{502, fmt.Sprintf("UPSTREAM_ERROR: %s", err)} } s.nsqadmin.logf(LOG_WARN, "%s", err) messages = append(messages, pe.Error()) } topicStats, _, err := s.ci.GetNSQDStats(producers, topicName, "", false) if err != nil { pe, ok := err.(clusterinfo.PartialErr) if !ok { s.nsqadmin.logf(LOG_ERROR, "failed to get topic metadata - %s", err) return nil, http_api.Err{502, fmt.Sprintf("UPSTREAM_ERROR: %s", err)} } s.nsqadmin.logf(LOG_WARN, "%s", err) messages = append(messages, pe.Error()) } allNodesTopicStats := &clusterinfo.TopicStats{TopicName: topicName} for _, t := range topicStats { allNodesTopicStats.Add(t) } return struct { *clusterinfo.TopicStats Message string `json:"message"` }{allNodesTopicStats, maybeWarnMsg(messages)}, nil } func (s *httpServer) channelHandler(w http.ResponseWriter, req *http.Request, ps httprouter.Params) (interface{}, error) { var messages []string topicName := ps.ByName("topic") channelName := ps.ByName("channel") producers, err := s.ci.GetTopicProducers(topicName, s.nsqadmin.getOpts().NSQLookupdHTTPAddresses, s.nsqadmin.getOpts().NSQDHTTPAddresses) if err != nil { pe, ok := err.(clusterinfo.PartialErr) if !ok { s.nsqadmin.logf(LOG_ERROR, "failed to get topic producers - %s", err) return nil, http_api.Err{502, fmt.Sprintf("UPSTREAM_ERROR: %s", err)} } s.nsqadmin.logf(LOG_WARN, "%s", err) messages = append(messages, pe.Error()) } _, channelStats, err := s.ci.GetNSQDStats(producers, topicName, channelName, true) if err != nil { pe, ok := err.(clusterinfo.PartialErr) if !ok { s.nsqadmin.logf(LOG_ERROR, "failed to get channel metadata - %s", err) return nil, http_api.Err{502, fmt.Sprintf("UPSTREAM_ERROR: %s", err)} } s.nsqadmin.logf(LOG_WARN, "%s", err) messages = append(messages, pe.Error()) } sort.Sort(clusterinfo.ClientStatsByNodeTopology{channelStats[channelName].Clients}) return struct { *clusterinfo.ChannelStats Message string `json:"message"` }{channelStats[channelName], maybeWarnMsg(messages)}, nil } func (s *httpServer) nodesHandler(w http.ResponseWriter, req *http.Request, ps httprouter.Params) (interface{}, error) { var messages []string producers, err := s.ci.GetProducers(s.nsqadmin.getOpts().NSQLookupdHTTPAddresses, s.nsqadmin.getOpts().NSQDHTTPAddresses) if err != nil { pe, ok := err.(clusterinfo.PartialErr) if !ok { s.nsqadmin.logf(LOG_ERROR, "failed to get nodes - %s", err) return nil, http_api.Err{502, fmt.Sprintf("UPSTREAM_ERROR: %s", err)} } s.nsqadmin.logf(LOG_WARN, "%s", err) messages = append(messages, pe.Error()) } return struct { Nodes clusterinfo.Producers `json:"nodes"` Message string `json:"message"` }{producers, maybeWarnMsg(messages)}, nil } func (s *httpServer) nodeHandler(w http.ResponseWriter, req *http.Request, ps httprouter.Params) (interface{}, error) { var messages []string node := ps.ByName("node") producers, err := s.ci.GetProducers(s.nsqadmin.getOpts().NSQLookupdHTTPAddresses, s.nsqadmin.getOpts().NSQDHTTPAddresses) if err != nil { pe, ok := err.(clusterinfo.PartialErr) if !ok { s.nsqadmin.logf(LOG_ERROR, "failed to get producers - %s", err) return nil, http_api.Err{502, fmt.Sprintf("UPSTREAM_ERROR: %s", err)} } s.nsqadmin.logf(LOG_WARN, "%s", err) messages = append(messages, pe.Error()) } producer := producers.Search(node) if producer == nil { return nil, http_api.Err{404, "NODE_NOT_FOUND"} } topicStats, _, err := s.ci.GetNSQDStats(clusterinfo.Producers{producer}, "", "", true) if err != nil { s.nsqadmin.logf(LOG_ERROR, "failed to get nsqd stats - %s", err) return nil, http_api.Err{502, fmt.Sprintf("UPSTREAM_ERROR: %s", err)} } var totalClients int64 var totalMessages int64 for _, ts := range topicStats { for _, cs := range ts.Channels { totalClients += int64(len(cs.Clients)) } totalMessages += ts.MessageCount } return struct { Node string `json:"node"` TopicStats []*clusterinfo.TopicStats `json:"topics"` TotalMessages int64 `json:"total_messages"` TotalClients int64 `json:"total_clients"` Message string `json:"message"` }{ Node: node, TopicStats: topicStats, TotalMessages: totalMessages, TotalClients: totalClients, Message: maybeWarnMsg(messages), }, nil } func (s *httpServer) tombstoneNodeForTopicHandler(w http.ResponseWriter, req *http.Request, ps httprouter.Params) (interface{}, error) { var messages []string if !s.isAuthorizedAdminRequest(req) { return nil, http_api.Err{403, "FORBIDDEN"} } node := ps.ByName("node") var body struct { Topic string `json:"topic"` } err := json.NewDecoder(req.Body).Decode(&body) if err != nil { return nil, http_api.Err{400, "INVALID_BODY"} } if !protocol.IsValidTopicName(body.Topic) { return nil, http_api.Err{400, "INVALID_TOPIC"} } err = s.ci.TombstoneNodeForTopic(body.Topic, node, s.nsqadmin.getOpts().NSQLookupdHTTPAddresses) if err != nil { pe, ok := err.(clusterinfo.PartialErr) if !ok { s.nsqadmin.logf(LOG_ERROR, "failed to tombstone node for topic - %s", err) return nil, http_api.Err{502, fmt.Sprintf("UPSTREAM_ERROR: %s", err)} } s.nsqadmin.logf(LOG_WARN, "%s", err) messages = append(messages, pe.Error()) } s.notifyAdminAction("tombstone_topic_producer", body.Topic, "", node, req) return struct { Message string `json:"message"` }{maybeWarnMsg(messages)}, nil } func (s *httpServer) createTopicChannelHandler(w http.ResponseWriter, req *http.Request, ps httprouter.Params) (interface{}, error) { var messages []string var body struct { Topic string `json:"topic"` Channel string `json:"channel"` } if !s.isAuthorizedAdminRequest(req) { return nil, http_api.Err{403, "FORBIDDEN"} } err := json.NewDecoder(req.Body).Decode(&body) if err != nil { return nil, http_api.Err{400, err.Error()} } if !protocol.IsValidTopicName(body.Topic) { return nil, http_api.Err{400, "INVALID_TOPIC"} } if len(body.Channel) > 0 && !protocol.IsValidChannelName(body.Channel) { return nil, http_api.Err{400, "INVALID_CHANNEL"} } err = s.ci.CreateTopicChannel(body.Topic, body.Channel, s.nsqadmin.getOpts().NSQLookupdHTTPAddresses) if err != nil { pe, ok := err.(clusterinfo.PartialErr) if !ok { s.nsqadmin.logf(LOG_ERROR, "failed to create topic/channel - %s", err) return nil, http_api.Err{502, fmt.Sprintf("UPSTREAM_ERROR: %s", err)} } s.nsqadmin.logf(LOG_WARN, "%s", err) messages = append(messages, pe.Error()) } s.notifyAdminAction("create_topic", body.Topic, "", "", req) if len(body.Channel) > 0 { s.notifyAdminAction("create_channel", body.Topic, body.Channel, "", req) } return struct { Message string `json:"message"` }{maybeWarnMsg(messages)}, nil } func (s *httpServer) deleteTopicHandler(w http.ResponseWriter, req *http.Request, ps httprouter.Params) (interface{}, error) { var messages []string if !s.isAuthorizedAdminRequest(req) { return nil, http_api.Err{403, "FORBIDDEN"} } topicName := ps.ByName("topic") err := s.ci.DeleteTopic(topicName, s.nsqadmin.getOpts().NSQLookupdHTTPAddresses, s.nsqadmin.getOpts().NSQDHTTPAddresses) if err != nil { pe, ok := err.(clusterinfo.PartialErr) if !ok { s.nsqadmin.logf(LOG_ERROR, "failed to delete topic - %s", err) return nil, http_api.Err{502, fmt.Sprintf("UPSTREAM_ERROR: %s", err)} } s.nsqadmin.logf(LOG_WARN, "%s", err) messages = append(messages, pe.Error()) } s.notifyAdminAction("delete_topic", topicName, "", "", req) return struct { Message string `json:"message"` }{maybeWarnMsg(messages)}, nil } func (s *httpServer) deleteChannelHandler(w http.ResponseWriter, req *http.Request, ps httprouter.Params) (interface{}, error) { var messages []string if !s.isAuthorizedAdminRequest(req) { return nil, http_api.Err{403, "FORBIDDEN"} } topicName := ps.ByName("topic") channelName := ps.ByName("channel") err := s.ci.DeleteChannel(topicName, channelName, s.nsqadmin.getOpts().NSQLookupdHTTPAddresses, s.nsqadmin.getOpts().NSQDHTTPAddresses) if err != nil { pe, ok := err.(clusterinfo.PartialErr) if !ok { s.nsqadmin.logf(LOG_ERROR, "failed to delete channel - %s", err) return nil, http_api.Err{502, fmt.Sprintf("UPSTREAM_ERROR: %s", err)} } s.nsqadmin.logf(LOG_WARN, "%s", err) messages = append(messages, pe.Error()) } s.notifyAdminAction("delete_channel", topicName, channelName, "", req) return struct { Message string `json:"message"` }{maybeWarnMsg(messages)}, nil } func (s *httpServer) topicActionHandler(w http.ResponseWriter, req *http.Request, ps httprouter.Params) (interface{}, error) { topicName := ps.ByName("topic") return s.topicChannelAction(req, topicName, "") } func (s *httpServer) channelActionHandler(w http.ResponseWriter, req *http.Request, ps httprouter.Params) (interface{}, error) { topicName := ps.ByName("topic") channelName := ps.ByName("channel") return s.topicChannelAction(req, topicName, channelName) } func (s *httpServer) topicChannelAction(req *http.Request, topicName string, channelName string) (interface{}, error) { var messages []string var body struct { Action string `json:"action"` } if !s.isAuthorizedAdminRequest(req) { return nil, http_api.Err{403, "FORBIDDEN"} } err := json.NewDecoder(req.Body).Decode(&body) if err != nil { return nil, http_api.Err{400, err.Error()} } switch body.Action { case "pause": if channelName != "" { err = s.ci.PauseChannel(topicName, channelName, s.nsqadmin.getOpts().NSQLookupdHTTPAddresses, s.nsqadmin.getOpts().NSQDHTTPAddresses) s.notifyAdminAction("pause_channel", topicName, channelName, "", req) } else { err = s.ci.PauseTopic(topicName, s.nsqadmin.getOpts().NSQLookupdHTTPAddresses, s.nsqadmin.getOpts().NSQDHTTPAddresses) s.notifyAdminAction("pause_topic", topicName, "", "", req) } case "unpause": if channelName != "" { err = s.ci.UnPauseChannel(topicName, channelName, s.nsqadmin.getOpts().NSQLookupdHTTPAddresses, s.nsqadmin.getOpts().NSQDHTTPAddresses) s.notifyAdminAction("unpause_channel", topicName, channelName, "", req) } else { err = s.ci.UnPauseTopic(topicName, s.nsqadmin.getOpts().NSQLookupdHTTPAddresses, s.nsqadmin.getOpts().NSQDHTTPAddresses) s.notifyAdminAction("unpause_topic", topicName, "", "", req) } case "empty": if channelName != "" { err = s.ci.EmptyChannel(topicName, channelName, s.nsqadmin.getOpts().NSQLookupdHTTPAddresses, s.nsqadmin.getOpts().NSQDHTTPAddresses) s.notifyAdminAction("empty_channel", topicName, channelName, "", req) } else { err = s.ci.EmptyTopic(topicName, s.nsqadmin.getOpts().NSQLookupdHTTPAddresses, s.nsqadmin.getOpts().NSQDHTTPAddresses) s.notifyAdminAction("empty_topic", topicName, "", "", req) } default: return nil, http_api.Err{400, "INVALID_ACTION"} } if err != nil { pe, ok := err.(clusterinfo.PartialErr) if !ok { s.nsqadmin.logf(LOG_ERROR, "failed to %s topic/channel - %s", body.Action, err) return nil, http_api.Err{502, fmt.Sprintf("UPSTREAM_ERROR: %s", err)} } s.nsqadmin.logf(LOG_WARN, "%s", err) messages = append(messages, pe.Error()) } return struct { Message string `json:"message"` }{maybeWarnMsg(messages)}, nil } type counterStats struct { Node string `json:"node"` TopicName string `json:"topic_name"` ChannelName string `json:"channel_name"` MessageCount int64 `json:"message_count"` } func (s *httpServer) counterHandler(w http.ResponseWriter, req *http.Request, ps httprouter.Params) (interface{}, error) { var messages []string stats := make(map[string]*counterStats) producers, err := s.ci.GetProducers(s.nsqadmin.getOpts().NSQLookupdHTTPAddresses, s.nsqadmin.getOpts().NSQDHTTPAddresses) if err != nil { pe, ok := err.(clusterinfo.PartialErr) if !ok { s.nsqadmin.logf(LOG_ERROR, "failed to get counter producer list - %s", err) return nil, http_api.Err{502, fmt.Sprintf("UPSTREAM_ERROR: %s", err)} } s.nsqadmin.logf(LOG_WARN, "%s", err) messages = append(messages, pe.Error()) } _, channelStats, err := s.ci.GetNSQDStats(producers, "", "", false) if err != nil { pe, ok := err.(clusterinfo.PartialErr) if !ok { s.nsqadmin.logf(LOG_ERROR, "failed to get nsqd stats - %s", err) return nil, http_api.Err{502, fmt.Sprintf("UPSTREAM_ERROR: %s", err)} } s.nsqadmin.logf(LOG_WARN, "%s", err) messages = append(messages, pe.Error()) } for _, channelStats := range channelStats { for _, hostChannelStats := range channelStats.NodeStats { key := fmt.Sprintf("%s:%s:%s", channelStats.TopicName, channelStats.ChannelName, hostChannelStats.Node) s, ok := stats[key] if !ok { s = &counterStats{ Node: hostChannelStats.Node, TopicName: channelStats.TopicName, ChannelName: channelStats.ChannelName, } stats[key] = s } s.MessageCount += hostChannelStats.MessageCount } } return struct { Stats map[string]*counterStats `json:"stats"` Message string `json:"message"` }{stats, maybeWarnMsg(messages)}, nil } func (s *httpServer) graphiteHandler(w http.ResponseWriter, req *http.Request, ps httprouter.Params) (interface{}, error) { reqParams, err := http_api.NewReqParams(req) if err != nil { return nil, http_api.Err{400, "INVALID_REQUEST"} } metric, err := reqParams.Get("metric") if err != nil || metric != "rate" { return nil, http_api.Err{400, "INVALID_ARG_METRIC"} } target, err := reqParams.Get("target") if err != nil { return nil, http_api.Err{400, "INVALID_ARG_TARGET"} } params := url.Values{} params.Set("from", fmt.Sprintf("-%dsec", s.nsqadmin.getOpts().StatsdInterval*2/time.Second)) params.Set("until", fmt.Sprintf("-%dsec", s.nsqadmin.getOpts().StatsdInterval/time.Second)) params.Set("format", "json") params.Set("target", target) query := fmt.Sprintf("/render?%s", params.Encode()) url := s.nsqadmin.getOpts().GraphiteURL + query s.nsqadmin.logf(LOG_INFO, "GRAPHITE: %s", url) var response []struct { Target string `json:"target"` DataPoints [][]*float64 `json:"datapoints"` } err = s.client.GETV1(url, &response) if err != nil { s.nsqadmin.logf(LOG_ERROR, "graphite request failed - %s", err) return nil, http_api.Err{500, "INTERNAL_ERROR"} } var rateStr string rate := *response[0].DataPoints[0][0] if rate < 0 { rateStr = "N/A" } else { rateDivisor := s.nsqadmin.getOpts().StatsdInterval / time.Second rateStr = fmt.Sprintf("%.2f", rate/float64(rateDivisor)) } return struct { Rate string `json:"rate"` }{rateStr}, nil } func (s *httpServer) doConfig(w http.ResponseWriter, req *http.Request, ps httprouter.Params) (interface{}, error) { opt := ps.ByName("opt") allowConfigFromCIDR := s.nsqadmin.getOpts().AllowConfigFromCIDR if allowConfigFromCIDR != "" { _, ipnet, _ := net.ParseCIDR(allowConfigFromCIDR) addr, _, err := net.SplitHostPort(req.RemoteAddr) if err != nil { s.nsqadmin.logf(LOG_ERROR, "failed to parse RemoteAddr %s", req.RemoteAddr) return nil, http_api.Err{400, "INVALID_REMOTE_ADDR"} } ip := net.ParseIP(addr) if ip == nil { s.nsqadmin.logf(LOG_ERROR, "failed to parse RemoteAddr %s", req.RemoteAddr) return nil, http_api.Err{400, "INVALID_REMOTE_ADDR"} } if !ipnet.Contains(ip) { return nil, http_api.Err{403, "FORBIDDEN"} } } if req.Method == "PUT" { // add 1 so that it's greater than our max when we test for it // (LimitReader returns a "fake" EOF) readMax := int64(1024*1024 + 1) body, err := io.ReadAll(io.LimitReader(req.Body, readMax)) if err != nil { return nil, http_api.Err{500, "INTERNAL_ERROR"} } if int64(len(body)) == readMax || len(body) == 0 { return nil, http_api.Err{413, "INVALID_VALUE"} } opts := *s.nsqadmin.getOpts() switch opt { case "nsqlookupd_http_addresses": err := json.Unmarshal(body, &opts.NSQLookupdHTTPAddresses) if err != nil { return nil, http_api.Err{400, "INVALID_VALUE"} } case "log_level": logLevelStr := string(body) logLevel, err := lg.ParseLogLevel(logLevelStr) if err != nil { return nil, http_api.Err{400, "INVALID_VALUE"} } opts.LogLevel = logLevel default: return nil, http_api.Err{400, "INVALID_OPTION"} } s.nsqadmin.swapOpts(&opts) } v, ok := getOptByCfgName(s.nsqadmin.getOpts(), opt) if !ok { return nil, http_api.Err{400, "INVALID_OPTION"} } return v, nil } func (s *httpServer) isAuthorizedAdminRequest(req *http.Request) bool { adminUsers := s.nsqadmin.getOpts().AdminUsers if len(adminUsers) == 0 { return true } aclHTTPHeader := s.nsqadmin.getOpts().ACLHTTPHeader user := req.Header.Get(aclHTTPHeader) for _, v := range adminUsers { if v == user { return true } } return false } func getOptByCfgName(opts interface{}, name string) (interface{}, bool) { val := reflect.ValueOf(opts).Elem() typ := val.Type() for i := 0; i < typ.NumField(); i++ { field := typ.Field(i) flagName := field.Tag.Get("flag") cfgName := field.Tag.Get("cfg") if flagName == "" { continue } if cfgName == "" { cfgName = strings.Replace(flagName, "-", "_", -1) } if name != cfgName { continue } return val.FieldByName(field.Name).Interface(), true } return nil, false } ================================================ FILE: nsqadmin/http_test.go ================================================ package nsqadmin import ( "bytes" "encoding/json" "fmt" "io" "net" "net/http" "os" "strconv" "testing" "time" "github.com/nsqio/nsq/internal/clusterinfo" "github.com/nsqio/nsq/internal/test" "github.com/nsqio/nsq/internal/version" "github.com/nsqio/nsq/nsqd" "github.com/nsqio/nsq/nsqlookupd" ) type TopicsDoc struct { Topics []interface{} `json:"topics"` } type TopicStatsDoc struct { *clusterinfo.TopicStats Message string `json:"message"` } type NodesDoc struct { Nodes clusterinfo.Producers `json:"nodes"` Message string `json:"message"` } type NodeStatsDoc struct { Node string `json:"node"` TopicStats []*clusterinfo.TopicStats `json:"topics"` TotalMessages int64 `json:"total_messages"` TotalClients int64 `json:"total_clients"` Message string `json:"message"` } type ChannelStatsDoc struct { *clusterinfo.ChannelStats Message string `json:"message"` } func mustStartNSQLookupd(opts *nsqlookupd.Options) (*net.TCPAddr, *net.TCPAddr, *nsqlookupd.NSQLookupd) { opts.TCPAddress = "127.0.0.1:0" opts.HTTPAddress = "127.0.0.1:0" lookupd, err := nsqlookupd.New(opts) if err != nil { panic(err) } go func() { err := lookupd.Main() if err != nil { panic(err) } }() return lookupd.RealTCPAddr(), lookupd.RealHTTPAddr(), lookupd } func bootstrapNSQCluster(t *testing.T) (string, []*nsqd.NSQD, []*nsqlookupd.NSQLookupd, *NSQAdmin) { return bootstrapNSQClusterWithAuth(t, false) } func bootstrapNSQClusterWithAuth(t *testing.T, withAuth bool) (string, []*nsqd.NSQD, []*nsqlookupd.NSQLookupd, *NSQAdmin) { lgr := test.NewTestLogger(t) nsqlookupdOpts := nsqlookupd.NewOptions() nsqlookupdOpts.TCPAddress = "127.0.0.1:0" nsqlookupdOpts.HTTPAddress = "127.0.0.1:0" nsqlookupdOpts.BroadcastAddress = "127.0.0.1" nsqlookupdOpts.Logger = lgr nsqlookupd1, err := nsqlookupd.New(nsqlookupdOpts) if err != nil { panic(err) } go func() { err := nsqlookupd1.Main() if err != nil { panic(err) } }() time.Sleep(100 * time.Millisecond) nsqdOpts := nsqd.NewOptions() nsqdOpts.TCPAddress = "127.0.0.1:0" nsqdOpts.HTTPAddress = "127.0.0.1:0" nsqdOpts.BroadcastAddress = "127.0.0.1" nsqdOpts.NSQLookupdTCPAddresses = []string{nsqlookupd1.RealTCPAddr().String()} nsqdOpts.Logger = lgr tmpDir, err := os.MkdirTemp("", "nsq-test-") if err != nil { panic(err) } nsqdOpts.DataPath = tmpDir nsqd1, err := nsqd.New(nsqdOpts) if err != nil { panic(err) } go func() { err := nsqd1.Main() if err != nil { panic(err) } }() nsqadminOpts := NewOptions() nsqadminOpts.HTTPAddress = "127.0.0.1:0" nsqadminOpts.NSQLookupdHTTPAddresses = []string{nsqlookupd1.RealHTTPAddr().String()} nsqadminOpts.Logger = lgr if withAuth { nsqadminOpts.AdminUsers = []string{"matt"} } nsqadmin1, err := New(nsqadminOpts) if err != nil { panic(err) } go func() { err := nsqadmin1.Main() if err != nil { panic(err) } }() time.Sleep(100 * time.Millisecond) return tmpDir, []*nsqd.NSQD{nsqd1}, []*nsqlookupd.NSQLookupd{nsqlookupd1}, nsqadmin1 } func TestPing(t *testing.T) { dataPath, nsqds, nsqlookupds, nsqadmin1 := bootstrapNSQCluster(t) defer os.RemoveAll(dataPath) defer nsqds[0].Exit() defer nsqlookupds[0].Exit() defer nsqadmin1.Exit() client := http.Client{} url := fmt.Sprintf("http://%s/ping", nsqadmin1.RealHTTPAddr()) req, _ := http.NewRequest("GET", url, nil) resp, err := client.Do(req) test.Nil(t, err) test.Equal(t, 200, resp.StatusCode) body, _ := io.ReadAll(resp.Body) resp.Body.Close() test.Equal(t, []byte("OK"), body) } func TestHTTPTopicsGET(t *testing.T) { dataPath, nsqds, nsqlookupds, nsqadmin1 := bootstrapNSQCluster(t) defer os.RemoveAll(dataPath) defer nsqds[0].Exit() defer nsqlookupds[0].Exit() defer nsqadmin1.Exit() topicName := "test_topics_get" + strconv.Itoa(int(time.Now().Unix())) nsqds[0].GetTopic(topicName) time.Sleep(100 * time.Millisecond) client := http.Client{} url := fmt.Sprintf("http://%s/api/topics", nsqadmin1.RealHTTPAddr()) req, _ := http.NewRequest("GET", url, nil) resp, err := client.Do(req) test.Nil(t, err) test.Equal(t, 200, resp.StatusCode) body, _ := io.ReadAll(resp.Body) resp.Body.Close() t.Logf("%s", body) tr := TopicsDoc{} err = json.Unmarshal(body, &tr) test.Nil(t, err) test.Equal(t, 1, len(tr.Topics)) test.Equal(t, topicName, tr.Topics[0]) } func TestHTTPTopicGET(t *testing.T) { dataPath, nsqds, nsqlookupds, nsqadmin1 := bootstrapNSQCluster(t) defer os.RemoveAll(dataPath) defer nsqds[0].Exit() defer nsqlookupds[0].Exit() defer nsqadmin1.Exit() topicName := "test_topic_get" + strconv.Itoa(int(time.Now().Unix())) nsqds[0].GetTopic(topicName) time.Sleep(100 * time.Millisecond) client := http.Client{} url := fmt.Sprintf("http://%s/api/topics/%s", nsqadmin1.RealHTTPAddr(), topicName) req, _ := http.NewRequest("GET", url, nil) resp, err := client.Do(req) test.Nil(t, err) test.Equal(t, 200, resp.StatusCode) body, _ := io.ReadAll(resp.Body) resp.Body.Close() t.Logf("%s", body) ts := TopicStatsDoc{} err = json.Unmarshal(body, &ts) test.Nil(t, err) test.Equal(t, topicName, ts.TopicName) test.Equal(t, 0, int(ts.Depth)) test.Equal(t, 0, int(ts.MemoryDepth)) test.Equal(t, 0, int(ts.BackendDepth)) test.Equal(t, 0, int(ts.MessageCount)) test.Equal(t, false, ts.Paused) } func TestHTTPNodesGET(t *testing.T) { dataPath, nsqds, nsqlookupds, nsqadmin1 := bootstrapNSQCluster(t) defer os.RemoveAll(dataPath) defer nsqds[0].Exit() defer nsqlookupds[0].Exit() defer nsqadmin1.Exit() time.Sleep(100 * time.Millisecond) client := http.Client{} url := fmt.Sprintf("http://%s/api/nodes", nsqadmin1.RealHTTPAddr()) req, _ := http.NewRequest("GET", url, nil) resp, err := client.Do(req) test.Nil(t, err) test.Equal(t, 200, resp.StatusCode) body, _ := io.ReadAll(resp.Body) resp.Body.Close() hostname, _ := os.Hostname() t.Logf("%s", body) ns := NodesDoc{} err = json.Unmarshal(body, &ns) test.Nil(t, err) test.Equal(t, 1, len(ns.Nodes)) testNode := ns.Nodes[0] test.Equal(t, hostname, testNode.Hostname) test.Equal(t, "127.0.0.1", testNode.BroadcastAddress) test.Equal(t, nsqds[0].RealTCPAddr().(*net.TCPAddr).Port, testNode.TCPPort) test.Equal(t, nsqds[0].RealHTTPAddr().(*net.TCPAddr).Port, testNode.HTTPPort) test.Equal(t, version.Binary, testNode.Version) test.Equal(t, 0, len(testNode.Topics)) } func TestHTTPChannelGET(t *testing.T) { dataPath, nsqds, nsqlookupds, nsqadmin1 := bootstrapNSQCluster(t) defer os.RemoveAll(dataPath) defer nsqds[0].Exit() defer nsqlookupds[0].Exit() defer nsqadmin1.Exit() topicName := "test_channel_get" + strconv.Itoa(int(time.Now().Unix())) topic := nsqds[0].GetTopic(topicName) topic.GetChannel("ch") time.Sleep(100 * time.Millisecond) client := http.Client{} url := fmt.Sprintf("http://%s/api/topics/%s/ch", nsqadmin1.RealHTTPAddr(), topicName) req, _ := http.NewRequest("GET", url, nil) resp, err := client.Do(req) test.Nil(t, err) test.Equal(t, 200, resp.StatusCode) body, _ := io.ReadAll(resp.Body) resp.Body.Close() t.Logf("%s", body) cs := ChannelStatsDoc{} err = json.Unmarshal(body, &cs) test.Nil(t, err) test.Equal(t, topicName, cs.TopicName) test.Equal(t, "ch", cs.ChannelName) test.Equal(t, 0, int(cs.Depth)) test.Equal(t, 0, int(cs.MemoryDepth)) test.Equal(t, 0, int(cs.BackendDepth)) test.Equal(t, 0, int(cs.MessageCount)) test.Equal(t, false, cs.Paused) test.Equal(t, 0, int(cs.InFlightCount)) test.Equal(t, 0, int(cs.DeferredCount)) test.Equal(t, 0, int(cs.RequeueCount)) test.Equal(t, 0, int(cs.TimeoutCount)) test.Equal(t, 0, len(cs.Clients)) } func TestHTTPNodesSingleGET(t *testing.T) { dataPath, nsqds, nsqlookupds, nsqadmin1 := bootstrapNSQCluster(t) defer os.RemoveAll(dataPath) defer nsqds[0].Exit() defer nsqlookupds[0].Exit() defer nsqadmin1.Exit() topicName := "test_nodes_single_get" + strconv.Itoa(int(time.Now().Unix())) topic := nsqds[0].GetTopic(topicName) topic.GetChannel("ch") time.Sleep(100 * time.Millisecond) client := http.Client{} url := fmt.Sprintf("http://%s/api/nodes/%s", nsqadmin1.RealHTTPAddr(), nsqds[0].RealHTTPAddr().String()) req, _ := http.NewRequest("GET", url, nil) resp, err := client.Do(req) test.Nil(t, err) test.Equal(t, 200, resp.StatusCode) body, _ := io.ReadAll(resp.Body) resp.Body.Close() t.Logf("%s", body) ns := NodeStatsDoc{} err = json.Unmarshal(body, &ns) test.Nil(t, err) test.Equal(t, nsqds[0].RealHTTPAddr().String(), ns.Node) test.Equal(t, 1, len(ns.TopicStats)) testTopic := ns.TopicStats[0] test.Equal(t, topicName, testTopic.TopicName) test.Equal(t, 0, int(testTopic.Depth)) test.Equal(t, 0, int(testTopic.MemoryDepth)) test.Equal(t, 0, int(testTopic.BackendDepth)) test.Equal(t, 0, int(testTopic.MessageCount)) test.Equal(t, false, testTopic.Paused) } func TestHTTPCreateTopicPOST(t *testing.T) { dataPath, nsqds, nsqlookupds, nsqadmin1 := bootstrapNSQCluster(t) defer os.RemoveAll(dataPath) defer nsqds[0].Exit() defer nsqlookupds[0].Exit() defer nsqadmin1.Exit() time.Sleep(100 * time.Millisecond) topicName := "test_create_topic_post" + strconv.Itoa(int(time.Now().Unix())) client := http.Client{} url := fmt.Sprintf("http://%s/api/topics", nsqadmin1.RealHTTPAddr()) body, _ := json.Marshal(map[string]interface{}{ "topic": topicName, }) req, _ := http.NewRequest("POST", url, bytes.NewBuffer(body)) resp, err := client.Do(req) test.Nil(t, err) test.Equal(t, 200, resp.StatusCode) resp.Body.Close() } func TestHTTPCreateTopicChannelPOST(t *testing.T) { dataPath, nsqds, nsqlookupds, nsqadmin1 := bootstrapNSQCluster(t) defer os.RemoveAll(dataPath) defer nsqds[0].Exit() defer nsqlookupds[0].Exit() defer nsqadmin1.Exit() time.Sleep(100 * time.Millisecond) topicName := "test_create_topic_channel_post" + strconv.Itoa(int(time.Now().Unix())) client := http.Client{} url := fmt.Sprintf("http://%s/api/topics", nsqadmin1.RealHTTPAddr()) body, _ := json.Marshal(map[string]interface{}{ "topic": topicName, "channel": "ch", }) req, _ := http.NewRequest("POST", url, bytes.NewBuffer(body)) resp, err := client.Do(req) test.Nil(t, err) test.Equal(t, 200, resp.StatusCode) resp.Body.Close() } func TestHTTPTombstoneTopicNodePOST(t *testing.T) { dataPath, nsqds, nsqlookupds, nsqadmin1 := bootstrapNSQCluster(t) defer os.RemoveAll(dataPath) defer nsqds[0].Exit() defer nsqlookupds[0].Exit() defer nsqadmin1.Exit() topicName := "test_tombstone_topic_node_post" + strconv.Itoa(int(time.Now().Unix())) nsqds[0].GetTopic(topicName) time.Sleep(100 * time.Millisecond) client := http.Client{} url := fmt.Sprintf("http://%s/api/nodes/%s", nsqadmin1.RealHTTPAddr(), nsqds[0].RealHTTPAddr()) body, _ := json.Marshal(map[string]interface{}{ "topic": topicName, }) req, _ := http.NewRequest("DELETE", url, bytes.NewBuffer(body)) resp, err := client.Do(req) test.Nil(t, err) test.Equal(t, 200, resp.StatusCode) resp.Body.Close() } func TestHTTPDeleteTopicPOST(t *testing.T) { dataPath, nsqds, nsqlookupds, nsqadmin1 := bootstrapNSQCluster(t) defer os.RemoveAll(dataPath) defer nsqds[0].Exit() defer nsqlookupds[0].Exit() defer nsqadmin1.Exit() topicName := "test_delete_topic_post" + strconv.Itoa(int(time.Now().Unix())) nsqds[0].GetTopic(topicName) time.Sleep(100 * time.Millisecond) client := http.Client{} url := fmt.Sprintf("http://%s/api/topics/%s", nsqadmin1.RealHTTPAddr(), topicName) req, _ := http.NewRequest("DELETE", url, nil) resp, err := client.Do(req) test.Nil(t, err) test.Equal(t, 200, resp.StatusCode) resp.Body.Close() } func TestHTTPDeleteChannelPOST(t *testing.T) { dataPath, nsqds, nsqlookupds, nsqadmin1 := bootstrapNSQCluster(t) defer os.RemoveAll(dataPath) defer nsqds[0].Exit() defer nsqlookupds[0].Exit() defer nsqadmin1.Exit() topicName := "test_delete_channel_post" + strconv.Itoa(int(time.Now().Unix())) topic := nsqds[0].GetTopic(topicName) topic.GetChannel("ch") time.Sleep(100 * time.Millisecond) client := http.Client{} url := fmt.Sprintf("http://%s/api/topics/%s/ch", nsqadmin1.RealHTTPAddr(), topicName) req, _ := http.NewRequest("DELETE", url, nil) resp, err := client.Do(req) test.Nil(t, err) test.Equal(t, 200, resp.StatusCode) resp.Body.Close() } func TestHTTPPauseTopicPOST(t *testing.T) { dataPath, nsqds, nsqlookupds, nsqadmin1 := bootstrapNSQCluster(t) defer os.RemoveAll(dataPath) defer nsqds[0].Exit() defer nsqlookupds[0].Exit() defer nsqadmin1.Exit() topicName := "test_pause_topic_post" + strconv.Itoa(int(time.Now().Unix())) nsqds[0].GetTopic(topicName) time.Sleep(100 * time.Millisecond) client := http.Client{} url := fmt.Sprintf("http://%s/api/topics/%s", nsqadmin1.RealHTTPAddr(), topicName) body, _ := json.Marshal(map[string]interface{}{ "action": "pause", }) req, _ := http.NewRequest("POST", url, bytes.NewBuffer(body)) resp, err := client.Do(req) test.Nil(t, err) _, _ = io.ReadAll(resp.Body) test.Equal(t, 200, resp.StatusCode) resp.Body.Close() url = fmt.Sprintf("http://%s/api/topics/%s", nsqadmin1.RealHTTPAddr(), topicName) body, _ = json.Marshal(map[string]interface{}{ "action": "unpause", }) req, _ = http.NewRequest("POST", url, bytes.NewBuffer(body)) resp, err = client.Do(req) test.Nil(t, err) test.Equal(t, 200, resp.StatusCode) resp.Body.Close() } func TestHTTPPauseChannelPOST(t *testing.T) { dataPath, nsqds, nsqlookupds, nsqadmin1 := bootstrapNSQCluster(t) defer os.RemoveAll(dataPath) defer nsqds[0].Exit() defer nsqlookupds[0].Exit() defer nsqadmin1.Exit() topicName := "test_pause_channel_post" + strconv.Itoa(int(time.Now().Unix())) topic := nsqds[0].GetTopic(topicName) topic.GetChannel("ch") time.Sleep(100 * time.Millisecond) client := http.Client{} url := fmt.Sprintf("http://%s/api/topics/%s/ch", nsqadmin1.RealHTTPAddr(), topicName) body, _ := json.Marshal(map[string]interface{}{ "action": "pause", }) req, _ := http.NewRequest("POST", url, bytes.NewBuffer(body)) resp, err := client.Do(req) test.Nil(t, err) _, _ = io.ReadAll(resp.Body) test.Equal(t, 200, resp.StatusCode) resp.Body.Close() url = fmt.Sprintf("http://%s/api/topics/%s/ch", nsqadmin1.RealHTTPAddr(), topicName) body, _ = json.Marshal(map[string]interface{}{ "action": "unpause", }) req, _ = http.NewRequest("POST", url, bytes.NewBuffer(body)) resp, err = client.Do(req) test.Nil(t, err) test.Equal(t, 200, resp.StatusCode) resp.Body.Close() } func TestHTTPEmptyTopicPOST(t *testing.T) { dataPath, nsqds, nsqlookupds, nsqadmin1 := bootstrapNSQCluster(t) defer os.RemoveAll(dataPath) defer nsqds[0].Exit() defer nsqlookupds[0].Exit() defer nsqadmin1.Exit() topicName := "test_empty_topic_post" + strconv.Itoa(int(time.Now().Unix())) topic := nsqds[0].GetTopic(topicName) topic.PutMessage(nsqd.NewMessage(nsqd.MessageID{}, []byte("1234"))) test.Equal(t, int64(1), topic.Depth()) time.Sleep(100 * time.Millisecond) client := http.Client{} url := fmt.Sprintf("http://%s/api/topics/%s", nsqadmin1.RealHTTPAddr(), topicName) body, _ := json.Marshal(map[string]interface{}{ "action": "empty", }) req, _ := http.NewRequest("POST", url, bytes.NewBuffer(body)) resp, err := client.Do(req) test.Nil(t, err) _, _ = io.ReadAll(resp.Body) test.Equal(t, 200, resp.StatusCode) resp.Body.Close() test.Equal(t, int64(0), topic.Depth()) } func TestHTTPEmptyChannelPOST(t *testing.T) { dataPath, nsqds, nsqlookupds, nsqadmin1 := bootstrapNSQCluster(t) defer os.RemoveAll(dataPath) defer nsqds[0].Exit() defer nsqlookupds[0].Exit() defer nsqadmin1.Exit() topicName := "test_empty_channel_post" + strconv.Itoa(int(time.Now().Unix())) topic := nsqds[0].GetTopic(topicName) channel := topic.GetChannel("ch") channel.PutMessage(nsqd.NewMessage(nsqd.MessageID{}, []byte("1234"))) time.Sleep(100 * time.Millisecond) test.Equal(t, int64(1), channel.Depth()) client := http.Client{} url := fmt.Sprintf("http://%s/api/topics/%s/ch", nsqadmin1.RealHTTPAddr(), topicName) body, _ := json.Marshal(map[string]interface{}{ "action": "empty", }) req, _ := http.NewRequest("POST", url, bytes.NewBuffer(body)) resp, err := client.Do(req) test.Nil(t, err) _, _ = io.ReadAll(resp.Body) test.Equal(t, 200, resp.StatusCode) resp.Body.Close() test.Equal(t, int64(0), channel.Depth()) } func TestHTTPconfig(t *testing.T) { dataPath, nsqds, nsqlookupds, nsqadmin1 := bootstrapNSQCluster(t) defer os.RemoveAll(dataPath) defer nsqds[0].Exit() defer nsqlookupds[0].Exit() defer nsqadmin1.Exit() lopts := nsqlookupd.NewOptions() lopts.Logger = test.NewTestLogger(t) lopts1 := *lopts _, _, lookupd1 := mustStartNSQLookupd(&lopts1) defer lookupd1.Exit() lopts2 := *lopts _, _, lookupd2 := mustStartNSQLookupd(&lopts2) defer lookupd2.Exit() url := fmt.Sprintf("http://%s/config/nsqlookupd_http_addresses", nsqadmin1.RealHTTPAddr()) resp, err := http.Get(url) test.Nil(t, err) defer resp.Body.Close() body, _ := io.ReadAll(resp.Body) test.Equal(t, 200, resp.StatusCode) origaddrs := fmt.Sprintf(`["%s"]`, nsqlookupds[0].RealHTTPAddr().String()) test.Equal(t, origaddrs, string(body)) client := http.Client{} addrs := fmt.Sprintf(`["%s","%s"]`, lookupd1.RealHTTPAddr().String(), lookupd2.RealHTTPAddr().String()) url = fmt.Sprintf("http://%s/config/nsqlookupd_http_addresses", nsqadmin1.RealHTTPAddr()) req, err := http.NewRequest("PUT", url, bytes.NewBuffer([]byte(addrs))) test.Nil(t, err) resp, err = client.Do(req) test.Nil(t, err) defer resp.Body.Close() body, _ = io.ReadAll(resp.Body) test.Equal(t, 200, resp.StatusCode) test.Equal(t, addrs, string(body)) url = fmt.Sprintf("http://%s/config/log_level", nsqadmin1.RealHTTPAddr()) req, err = http.NewRequest("PUT", url, bytes.NewBuffer([]byte(`fatal`))) test.Nil(t, err) resp, err = client.Do(req) test.Nil(t, err) defer resp.Body.Close() _, _ = io.ReadAll(resp.Body) test.Equal(t, 200, resp.StatusCode) test.Equal(t, LOG_FATAL, nsqadmin1.getOpts().LogLevel) url = fmt.Sprintf("http://%s/config/log_level", nsqadmin1.RealHTTPAddr()) req, err = http.NewRequest("PUT", url, bytes.NewBuffer([]byte(`bad`))) test.Nil(t, err) resp, err = client.Do(req) test.Nil(t, err) defer resp.Body.Close() _, _ = io.ReadAll(resp.Body) test.Equal(t, 400, resp.StatusCode) } func TestHTTPconfigCIDR(t *testing.T) { opts := NewOptions() opts.HTTPAddress = "127.0.0.1:0" opts.NSQLookupdHTTPAddresses = []string{"127.0.0.1:4161"} opts.Logger = test.NewTestLogger(t) opts.AllowConfigFromCIDR = "10.0.0.0/8" nsqadmin, err := New(opts) test.Nil(t, err) go func() { err := nsqadmin.Main() if err != nil { panic(err) } }() defer nsqadmin.Exit() time.Sleep(100 * time.Millisecond) url := fmt.Sprintf("http://%s/config/nsqlookupd_http_addresses", nsqadmin.RealHTTPAddr()) resp, err := http.Get(url) test.Nil(t, err) defer resp.Body.Close() _, _ = io.ReadAll(resp.Body) test.Equal(t, 403, resp.StatusCode) } ================================================ FILE: nsqadmin/logger.go ================================================ package nsqadmin import ( "github.com/nsqio/nsq/internal/lg" ) type Logger lg.Logger const ( LOG_DEBUG = lg.DEBUG LOG_INFO = lg.INFO LOG_WARN = lg.WARN LOG_ERROR = lg.ERROR LOG_FATAL = lg.FATAL ) func (n *NSQAdmin) logf(level lg.LogLevel, f string, args ...interface{}) { opts := n.getOpts() lg.Logf(opts.Logger, opts.LogLevel, level, f, args...) } ================================================ FILE: nsqadmin/notify.go ================================================ package nsqadmin import ( "encoding/base64" "net/http" "net/url" "os" "strings" "time" ) type AdminAction struct { Action string `json:"action"` Topic string `json:"topic"` Channel string `json:"channel,omitempty"` Node string `json:"node,omitempty"` Timestamp int64 `json:"timestamp"` User string `json:"user,omitempty"` RemoteIP string `json:"remote_ip"` UserAgent string `json:"user_agent"` URL string `json:"url"` // The URL of the HTTP request that triggered this action Via string `json:"via"` // the Hostname of the nsqadmin performing this action } func basicAuthUser(req *http.Request) string { s := strings.SplitN(req.Header.Get("Authorization"), " ", 2) if len(s) != 2 || s[0] != "Basic" { return "" } b, err := base64.StdEncoding.DecodeString(s[1]) if err != nil { return "" } pair := strings.SplitN(string(b), ":", 2) if len(pair) != 2 { return "" } return pair[0] } func (s *httpServer) notifyAdminAction(action, topic, channel, node string, req *http.Request) { if s.nsqadmin.getOpts().NotificationHTTPEndpoint == "" { return } via, _ := os.Hostname() u := url.URL{ Scheme: "http", Host: req.Host, Path: req.URL.Path, RawQuery: req.URL.RawQuery, } if req.TLS != nil || req.Header.Get("X-Scheme") == "https" { u.Scheme = "https" } a := &AdminAction{ Action: action, Topic: topic, Channel: channel, Node: node, Timestamp: time.Now().Unix(), User: basicAuthUser(req), RemoteIP: req.RemoteAddr, UserAgent: req.UserAgent(), URL: u.String(), Via: via, } // Perform all work in a new goroutine so this never blocks go func() { s.nsqadmin.notifications <- a }() } ================================================ FILE: nsqadmin/nsqadmin.go ================================================ package nsqadmin import ( "bytes" "crypto/tls" "crypto/x509" "encoding/json" "errors" "fmt" "log" "net" "net/http" "net/url" "os" "path" "sync" "sync/atomic" "github.com/nsqio/nsq/internal/http_api" "github.com/nsqio/nsq/internal/util" "github.com/nsqio/nsq/internal/version" ) type NSQAdmin struct { sync.RWMutex opts atomic.Value httpListener net.Listener waitGroup util.WaitGroupWrapper notifications chan *AdminAction graphiteURL *url.URL httpClientTLSConfig *tls.Config } func New(opts *Options) (*NSQAdmin, error) { if opts.Logger == nil { opts.Logger = log.New(os.Stderr, opts.LogPrefix, log.Ldate|log.Ltime|log.Lmicroseconds) } n := &NSQAdmin{ notifications: make(chan *AdminAction), } n.swapOpts(opts) if len(opts.NSQDHTTPAddresses) == 0 && len(opts.NSQLookupdHTTPAddresses) == 0 { return nil, errors.New("--nsqd-http-address or --lookupd-http-address required") } if len(opts.NSQDHTTPAddresses) != 0 && len(opts.NSQLookupdHTTPAddresses) != 0 { return nil, errors.New("use --nsqd-http-address or --lookupd-http-address not both") } if opts.HTTPClientTLSCert != "" && opts.HTTPClientTLSKey == "" { return nil, errors.New("--http-client-tls-key must be specified with --http-client-tls-cert") } if opts.HTTPClientTLSKey != "" && opts.HTTPClientTLSCert == "" { return nil, errors.New("--http-client-tls-cert must be specified with --http-client-tls-key") } n.httpClientTLSConfig = &tls.Config{ InsecureSkipVerify: opts.HTTPClientTLSInsecureSkipVerify, } if opts.HTTPClientTLSCert != "" && opts.HTTPClientTLSKey != "" { cert, err := tls.LoadX509KeyPair(opts.HTTPClientTLSCert, opts.HTTPClientTLSKey) if err != nil { return nil, fmt.Errorf("failed to LoadX509KeyPair %s, %s - %s", opts.HTTPClientTLSCert, opts.HTTPClientTLSKey, err) } n.httpClientTLSConfig.Certificates = []tls.Certificate{cert} } if opts.HTTPClientTLSRootCAFile != "" { tlsCertPool := x509.NewCertPool() caCertFile, err := os.ReadFile(opts.HTTPClientTLSRootCAFile) if err != nil { return nil, fmt.Errorf("failed to read TLS root CA file %s - %s", opts.HTTPClientTLSRootCAFile, err) } if !tlsCertPool.AppendCertsFromPEM(caCertFile) { return nil, fmt.Errorf("failed to AppendCertsFromPEM %s", opts.HTTPClientTLSRootCAFile) } n.httpClientTLSConfig.RootCAs = tlsCertPool } for _, address := range opts.NSQLookupdHTTPAddresses { _, err := net.ResolveTCPAddr("tcp", address) if err != nil { return nil, fmt.Errorf("failed to resolve --lookupd-http-address (%s) - %s", address, err) } } for _, address := range opts.NSQDHTTPAddresses { _, err := net.ResolveTCPAddr("tcp", address) if err != nil { return nil, fmt.Errorf("failed to resolve --nsqd-http-address (%s) - %s", address, err) } } if opts.ProxyGraphite { url, err := url.Parse(opts.GraphiteURL) if err != nil { return nil, fmt.Errorf("failed to parse --graphite-url (%s) - %s", opts.GraphiteURL, err) } n.graphiteURL = url } if opts.AllowConfigFromCIDR != "" { _, _, err := net.ParseCIDR(opts.AllowConfigFromCIDR) if err != nil { return nil, fmt.Errorf("failed to parse --allow-config-from-cidr (%s) - %s", opts.AllowConfigFromCIDR, err) } } opts.BasePath = normalizeBasePath(opts.BasePath) n.logf(LOG_INFO, version.String("nsqadmin")) var err error n.httpListener, err = net.Listen("tcp", n.getOpts().HTTPAddress) if err != nil { return nil, fmt.Errorf("listen (%s) failed - %s", n.getOpts().HTTPAddress, err) } return n, nil } func normalizeBasePath(p string) string { if len(p) == 0 { return "/" } // add leading slash if p[0] != '/' { p = "/" + p } return path.Clean(p) } func (n *NSQAdmin) getOpts() *Options { return n.opts.Load().(*Options) } func (n *NSQAdmin) swapOpts(opts *Options) { n.opts.Store(opts) } func (n *NSQAdmin) RealHTTPAddr() *net.TCPAddr { return n.httpListener.Addr().(*net.TCPAddr) } func (n *NSQAdmin) handleAdminActions() { for action := range n.notifications { content, err := json.Marshal(action) if err != nil { n.logf(LOG_ERROR, "failed to serialize admin action - %s", err) } httpclient := &http.Client{ Transport: http_api.NewDeadlineTransport(n.getOpts().HTTPClientConnectTimeout, n.getOpts().HTTPClientRequestTimeout), } n.logf(LOG_INFO, "POSTing notification to %s", n.getOpts().NotificationHTTPEndpoint) resp, err := httpclient.Post(n.getOpts().NotificationHTTPEndpoint, "application/json", bytes.NewBuffer(content)) if err != nil { n.logf(LOG_ERROR, "failed to POST notification - %s", err) } resp.Body.Close() } } func (n *NSQAdmin) Main() error { exitCh := make(chan error) var once sync.Once exitFunc := func(err error) { once.Do(func() { if err != nil { n.logf(LOG_FATAL, "%s", err) } exitCh <- err }) } httpServer := NewHTTPServer(n) n.waitGroup.Wrap(func() { exitFunc(http_api.Serve(n.httpListener, http_api.CompressHandler(httpServer), "HTTP", n.logf)) }) n.waitGroup.Wrap(n.handleAdminActions) err := <-exitCh return err } func (n *NSQAdmin) Exit() { if n.httpListener != nil { n.httpListener.Close() } close(n.notifications) n.waitGroup.Wait() } ================================================ FILE: nsqadmin/nsqadmin_test.go ================================================ package nsqadmin import ( "fmt" "net" "net/http" "net/url" "os" "testing" "github.com/nsqio/nsq/internal/lg" "github.com/nsqio/nsq/internal/test" "github.com/nsqio/nsq/nsqd" ) func TestNeitherNSQDAndNSQLookup(t *testing.T) { opts := NewOptions() opts.Logger = lg.NilLogger{} opts.HTTPAddress = "127.0.0.1:0" _, err := New(opts) test.NotNil(t, err) test.Equal(t, "--nsqd-http-address or --lookupd-http-address required", fmt.Sprintf("%s", err)) } func TestBothNSQDAndNSQLookup(t *testing.T) { opts := NewOptions() opts.Logger = lg.NilLogger{} opts.HTTPAddress = "127.0.0.1:0" opts.NSQLookupdHTTPAddresses = []string{"127.0.0.1:4161"} opts.NSQDHTTPAddresses = []string{"127.0.0.1:4151"} _, err := New(opts) test.NotNil(t, err) test.Equal(t, "use --nsqd-http-address or --lookupd-http-address not both", fmt.Sprintf("%s", err)) } func TestTLSHTTPClient(t *testing.T) { lgr := test.NewTestLogger(t) nsqdOpts := nsqd.NewOptions() nsqdOpts.TLSCert = "./test/server.pem" nsqdOpts.TLSKey = "./test/server.key" nsqdOpts.TLSRootCAFile = "./test/ca.pem" nsqdOpts.TLSClientAuthPolicy = "require-verify" nsqdOpts.Logger = lgr _, nsqdHTTPAddr, nsqd := mustStartNSQD(nsqdOpts) defer os.RemoveAll(nsqdOpts.DataPath) defer nsqd.Exit() opts := NewOptions() opts.HTTPAddress = "127.0.0.1:0" opts.NSQDHTTPAddresses = []string{nsqdHTTPAddr.String()} opts.HTTPClientTLSRootCAFile = "./test/ca.pem" opts.HTTPClientTLSCert = "./test/client.pem" opts.HTTPClientTLSKey = "./test/client.key" opts.Logger = lgr nsqadmin, err := New(opts) test.Nil(t, err) go func() { err := nsqadmin.Main() if err != nil { panic(err) } }() defer nsqadmin.Exit() httpAddr := nsqadmin.RealHTTPAddr() u := url.URL{ Scheme: "http", Host: httpAddr.String(), Path: "/api/nodes/" + nsqdHTTPAddr.String(), } resp, err := http.Get(u.String()) test.Nil(t, err) defer resp.Body.Close() test.Equal(t, resp.StatusCode < 500, true) } func mustStartNSQD(opts *nsqd.Options) (net.Addr, net.Addr, *nsqd.NSQD) { opts.TCPAddress = "127.0.0.1:0" opts.HTTPAddress = "127.0.0.1:0" opts.HTTPSAddress = "127.0.0.1:0" if opts.DataPath == "" { tmpDir, err := os.MkdirTemp("", "nsq-test-") if err != nil { panic(err) } opts.DataPath = tmpDir } nsqd, err := nsqd.New(opts) if err != nil { panic(err) } go func() { err := nsqd.Main() if err != nil { panic(err) } }() return nsqd.RealTCPAddr(), nsqd.RealHTTPAddr(), nsqd } ================================================ FILE: nsqadmin/options.go ================================================ package nsqadmin import ( "time" "github.com/nsqio/nsq/internal/lg" ) type Options struct { LogLevel lg.LogLevel `flag:"log-level"` LogPrefix string `flag:"log-prefix"` Logger Logger HTTPAddress string `flag:"http-address"` BasePath string `flag:"base-path"` DevStaticDir string `flag:"dev-static-dir"` GraphiteURL string `flag:"graphite-url"` ProxyGraphite bool `flag:"proxy-graphite"` StatsdPrefix string `flag:"statsd-prefix"` StatsdCounterFormat string `flag:"statsd-counter-format"` StatsdGaugeFormat string `flag:"statsd-gauge-format"` StatsdInterval time.Duration `flag:"statsd-interval"` NSQLookupdHTTPAddresses []string `flag:"lookupd-http-address" cfg:"nsqlookupd_http_addresses"` NSQDHTTPAddresses []string `flag:"nsqd-http-address" cfg:"nsqd_http_addresses"` HTTPClientConnectTimeout time.Duration `flag:"http-client-connect-timeout"` HTTPClientRequestTimeout time.Duration `flag:"http-client-request-timeout"` HTTPClientTLSInsecureSkipVerify bool `flag:"http-client-tls-insecure-skip-verify"` HTTPClientTLSRootCAFile string `flag:"http-client-tls-root-ca-file"` HTTPClientTLSCert string `flag:"http-client-tls-cert"` HTTPClientTLSKey string `flag:"http-client-tls-key"` AllowConfigFromCIDR string `flag:"allow-config-from-cidr"` NotificationHTTPEndpoint string `flag:"notification-http-endpoint"` ACLHTTPHeader string `flag:"acl-http-header"` AdminUsers []string `flag:"admin-user" cfg:"admin_users"` } func NewOptions() *Options { return &Options{ LogPrefix: "[nsqadmin] ", LogLevel: lg.INFO, HTTPAddress: "0.0.0.0:4171", BasePath: "/", StatsdPrefix: "nsq.%s", StatsdCounterFormat: "stats.counters.%s.count", StatsdGaugeFormat: "stats.gauges.%s", StatsdInterval: 60 * time.Second, HTTPClientConnectTimeout: 2 * time.Second, HTTPClientRequestTimeout: 5 * time.Second, AllowConfigFromCIDR: "127.0.0.1/8", ACLHTTPHeader: "X-Forwarded-User", AdminUsers: []string{}, } } ================================================ FILE: nsqadmin/package.json ================================================ { "name": "nsqadmin", "version": "0.3.0", "description": "operational dashboard for NSQ (https://nsq.io/)", "repository": { "type": "git", "url": "" }, "scripts": { "lint": "eslint static/js" }, "devDependencies": { "browserify": "^17.0.0", "eslint": "^8.52.0", "gulp": "^4.0.2", "gulp-clean": "^0.4.0", "gulp-cli": "^2.3.0", "gulp-dart-sass": "^1.0.2", "gulp-notify": "^4.0.0", "gulp-sourcemaps": "^3.0.0", "gulp-task-listing": "^1.1.0", "gulp-uglify": "^3.0.2", "handlebars": "^4.7.7", "hbsfy": "^2.8.1", "vinyl-buffer": "^1.0.1", "vinyl-source-stream": "^2.0.0" }, "dependencies": { "backbone": "^1.4.0", "bootbox": "^5.5.2", "bootstrap": "^3.4.1", "jquery": "^3.6.0", "moment": "^2.29.1", "underscore": "^1.13.1" }, "browserify": { "transform": [ "hbsfy" ] } } ================================================ FILE: nsqadmin/static/build/base.css ================================================ .red { color: #c30; } .red:hover { color: #f30; } .bold { font-weight: bold; } .graph-row td { text-align: center; } .image-preview { display: none; position: absolute; z-index: 100; height: 240px; width: 480px; } .white { color: #fff; } .bubblingG { text-align: center; width: 125px; height: 78px; } .bubblingG span { display: inline-block; vertical-align: middle; width: 16px; height: 16px; margin: 39px auto; background: #006FC4; -moz-border-radius: 79px; -moz-animation: bubblingG 0.9s infinite alternate; -webkit-border-radius: 79px; -webkit-animation: bubblingG 0.9s infinite alternate; -ms-border-radius: 79px; -ms-animation: bubblingG 0.9s infinite alternate; -o-border-radius: 79px; -o-animation: bubblingG 0.9s infinite alternate; border-radius: 79px; animation: bubblingG 0.9s infinite alternate; } #bubblingG_1 { -moz-animation-delay: 0s; -webkit-animation-delay: 0s; -ms-animation-delay: 0s; -o-animation-delay: 0s; animation-delay: 0s; } #bubblingG_2 { -moz-animation-delay: 0.27s; -webkit-animation-delay: 0.27s; -ms-animation-delay: 0.27s; -o-animation-delay: 0.27s; animation-delay: 0.27s; } #bubblingG_3 { -moz-animation-delay: 0.54s; -webkit-animation-delay: 0.54s; -ms-animation-delay: 0.54s; -o-animation-delay: 0.54s; animation-delay: 0.54s; } @-moz-keyframes bubblingG { 0% { width: 16px; height: 16px; background-color: #006FC4; -moz-transform: translateY(0); } 100% { width: 38px; height: 38px; background-color: #FFFFFF; -moz-transform: translateY(-33px); } } @-webkit-keyframes bubblingG { 0% { width: 16px; height: 16px; background-color: #006FC4; -webkit-transform: translateY(0); } 100% { width: 38px; height: 38px; background-color: #FFFFFF; -webkit-transform: translateY(-33px); } } @-ms-keyframes bubblingG { 0% { width: 16px; height: 16px; background-color: #006FC4; -ms-transform: translateY(0); } 100% { width: 38px; height: 38px; background-color: #FFFFFF; -ms-transform: translateY(-33px); } } @-o-keyframes bubblingG { 0% { width: 16px; height: 16px; background-color: #006FC4; -o-transform: translateY(0); } 100% { width: 38px; height: 38px; background-color: #FFFFFF; -o-transform: translateY(-33px); } } @keyframes bubblingG { 0% { width: 16px; height: 16px; background-color: #006FC4; transform: translateY(0); } 100% { width: 38px; height: 38px; background-color: #FFFFFF; transform: translateY(-33px); } } .bubblingG { position: absolute; left: 0; right: 0; top: 0; bottom: 0; margin: auto; max-width: 100%; max-height: 100%; overflow: auto; } .navbar-brand > img { width: 30px; height: 30px; margin-right: 5px; margin-top: -5px; display: inline; } .bg-zone-local { background-color: #ddffdd; } .bg-region-local { background-color: #fefec2; } .bg-global { background-color: white; } .popup { position: relative; display: inline-block; cursor: pointer; } /* The actual popup (appears on top) */ .popup .popuptext { visibility: hidden; width: 180px; height: 27px; background-color: white; color: #4b4b4b; text-align: center; border-radius: 6px; border-right: 1px solid #777; border-left: 1px solid #777; border-top: 1px solid #777; border-bottom: 1px solid #777; padding: 2px 0; position: absolute; z-index: 1; top: 60%; left: 50%; margin-left: -85%; } /* Popup arrow */ .popup .popuptext::after { content: ""; position: absolute; top: -5px; left: 50%; margin-left: -5px; width: 0; height: 0; border-left: 5px solid transparent; border-right: 5px solid transparent; border-bottom: 5px solid #777; } /* Toggle this class when clicking on the popup container (hide and show the popup) */ .popup .show { visibility: visible; -webkit-animation: fadeIn 1s; animation: fadeIn 1s; } /* Add animation (fade in the popup) */ @-webkit-keyframes fadeIn { from { opacity: 0; } to { opacity: 1; } } @keyframes fadeIn { from { opacity: 0; } to { opacity: 1; } } ================================================ FILE: nsqadmin/static/build/index.html ================================================ nsqadmin
================================================ FILE: nsqadmin/static/build/main.js ================================================ !function l(a,o,r){function i(e,n){if(!o[e]){if(!a[e]){var t="function"==typeof require&&require;if(!n&&t)return t(e,!0);if(s)return s(e,!0);throw(t=new Error("Cannot find module '"+e+"'")).code="MODULE_NOT_FOUND",t}t=o[e]={exports:{}},a[e][0].call(t.exports,function(n){return i(a[e][1][n]||n)},t,t.exports,l,a,o,r)}return o[e].exports}for(var s="function"==typeof require&&require,n=0;nthis.$items.length-1||n<0))return this.sliding?this.$element.one("slid.bs.carousel",function(){e.to(n)}):t==n?this.pause().cycle():this.slide(tdocument.documentElement.clientHeight;this.$element.css({paddingLeft:!this.bodyIsOverflowing&&n?this.scrollbarWidth:"",paddingRight:this.bodyIsOverflowing&&!n?this.scrollbarWidth:""})},r.prototype.resetAdjustments=function(){this.$element.css({paddingLeft:"",paddingRight:""})},r.prototype.checkScrollbar=function(){var n,e=window.innerWidth;e||(e=(n=document.documentElement.getBoundingClientRect()).right-Math.abs(n.left)),this.bodyIsOverflowing=document.body.clientWidth

'}),((o.prototype=a.extend({},a.fn.tooltip.Constructor.prototype)).constructor=o).prototype.getDefaults=function(){return o.DEFAULTS},o.prototype.setContent=function(){var n,e=this.tip(),t=this.getTitle(),l=this.getContent();this.options.html?(n=typeof l,this.options.sanitize&&(t=this.sanitizeHtml(t),"string"==n&&(l=this.sanitizeHtml(l))),e.find(".popover-title").html(t),e.find(".popover-content").children().detach().end()["string"==n?"html":"append"](l)):(e.find(".popover-title").text(t),e.find(".popover-content").children().detach().end().text(l)),e.removeClass("fade top bottom left right in"),e.find(".popover-title").html()||e.find(".popover-title").hide()},o.prototype.hasContent=function(){return this.getTitle()||this.getContent()},o.prototype.getContent=function(){var n=this.$element,e=this.options;return n.attr("data-content")||("function"==typeof e.content?e.content.call(n[0]):e.content)},o.prototype.arrow=function(){return this.$arrow=this.$arrow||this.tip().find(".arrow")};var n=a.fn.popover;a.fn.popover=function(l){return this.each(function(){var n=a(this),e=n.data("bs.popover"),t="object"==typeof l&&l;!e&&/destroy|hide/.test(l)||(e||n.data("bs.popover",e=new o(this,t)),"string"==typeof l&&e[l]())})},a.fn.popover.Constructor=o,a.fn.popover.noConflict=function(){return a.fn.popover=n,this}}(jQuery)},{}],10:[function(n,e,t){!function(a){"use strict";function l(n,e){this.$body=a(document.body),this.$scrollElement=a(n).is(document.body)?a(window):a(n),this.options=a.extend({},l.DEFAULTS,e),this.selector=(this.options.target||"")+" .nav li > a",this.offsets=[],this.targets=[],this.activeTarget=null,this.scrollHeight=0,this.$scrollElement.on("scroll.bs.scrollspy",a.proxy(this.process,this)),this.refresh(),this.process()}function e(t){return this.each(function(){var n=a(this),e=n.data("bs.scrollspy");e||n.data("bs.scrollspy",e=new l(this,"object"==typeof t&&t)),"string"==typeof t&&e[t]()})}l.VERSION="3.4.1",l.DEFAULTS={offset:10},l.prototype.getScrollHeight=function(){return this.$scrollElement[0].scrollHeight||Math.max(this.$body[0].scrollHeight,document.documentElement.scrollHeight)},l.prototype.refresh=function(){var n=this,t="offset",l=0;this.offsets=[],this.targets=[],this.scrollHeight=this.getScrollHeight(),a.isWindow(this.$scrollElement[0])||(t="position",l=this.$scrollElement.scrollTop()),this.$body.find(this.selector).map(function(){var n=a(this),e=n.data("target")||n.attr("href"),n=/^#./.test(e)&&a(e);return n&&n.length&&n.is(":visible")?[[n[t]().top+l,e]]:null}).sort(function(n,e){return n[0]-e[0]}).each(function(){n.offsets.push(this[0]),n.targets.push(this[1])})},l.prototype.process=function(){var n,e=this.$scrollElement.scrollTop()+this.options.offset,t=this.getScrollHeight(),l=this.options.offset+t-this.$scrollElement.height(),a=this.offsets,o=this.targets,r=this.activeTarget;if(this.scrollHeight!=t&&this.refresh(),l<=e)return r!=(n=o[o.length-1])&&this.activate(n);if(r&&e=a[n]&&(void 0===a[n+1]||e .active"),a=t&&r.support.transition&&(l.length&&l.hasClass("fade")||!!e.find("> .fade").length);function o(){l.removeClass("active").find("> .dropdown-menu > .active").removeClass("active").end().find('[data-toggle="tab"]').attr("aria-expanded",!1),n.addClass("active").find('[data-toggle="tab"]').attr("aria-expanded",!0),a?(n[0].offsetWidth,n.addClass("in")):n.removeClass("fade"),n.parent(".dropdown-menu").length&&n.closest("li.dropdown").addClass("active").end().find('[data-toggle="tab"]').attr("aria-expanded",!0),t&&t()}l.length&&a?l.one("bsTransitionEnd",o).emulateTransitionEnd(i.TRANSITION_DURATION):o(),l.removeClass("in")};var n=r.fn.tab;r.fn.tab=e,r.fn.tab.Constructor=i,r.fn.tab.noConflict=function(){return r.fn.tab=n,this};function t(n){n.preventDefault(),e.call(r(this),"show")}r(document).on("click.bs.tab.data-api",'[data-toggle="tab"]',t).on("click.bs.tab.data-api",'[data-toggle="pill"]',t)}(jQuery)},{}],12:[function(n,e,t){!function(d){"use strict";var l=["sanitize","whiteList","sanitizeFn"],m=["background","cite","href","itemtype","longdesc","poster","src","xlink:href"],n={"*":["class","dir","id","lang","role",/^aria-[\w-]*$/i],a:["target","href","title","rel"],area:[],b:[],br:[],col:[],code:[],div:[],em:[],hr:[],h1:[],h2:[],h3:[],h4:[],h5:[],h6:[],i:[],img:["src","alt","title","width","height"],li:[],ol:[],p:[],pre:[],s:[],small:[],span:[],sub:[],sup:[],strong:[],u:[],ul:[]},f=/^(?:(?:https?|mailto|ftp|tel|file):|[^&:/?#]*(?:[/?#]|$))/gi,g=/^data:(?:image\/(?:bmp|gif|jpeg|jpg|png|tiff|webp)|video\/(?:mpeg|mp4|ogg|webm)|audio\/(?:mp3|oga|ogg|opus));base64,[a-z0-9+/]+=*$/i;function a(n,e,t){if(0===n.length)return n;if(t&&"function"==typeof t)return t(n);if(!document.implementation||!document.implementation.createHTMLDocument)return n;t=document.implementation.createHTMLDocument("sanitization");t.body.innerHTML=n;for(var l=d.map(e,function(n,e){return e}),a=d(t.body).find("*"),o=0,r=a.length;o
',trigger:"hover focus",title:"",delay:0,html:!1,container:!1,viewport:{selector:"body",padding:0},sanitize:!0,sanitizeFn:null,whiteList:n},s.prototype.init=function(n,e,t){if(this.enabled=!0,this.type=n,this.$element=d(e),this.options=this.getOptions(t),this.$viewport=this.options.viewport&&d(document).find(d.isFunction(this.options.viewport)?this.options.viewport.call(this,this.$element):this.options.viewport.selector||this.options.viewport),this.inState={click:!1,hover:!1,focus:!1},this.$element[0]instanceof document.constructor&&!this.options.selector)throw new Error("`selector` option must be specified when initializing "+this.type+" on the window.document object!");for(var l=this.options.trigger.split(" "),a=l.length;a--;){var o,r=l[a];"click"==r?this.$element.on("click."+this.type,this.options.selector,d.proxy(this.toggle,this)):"manual"!=r&&(o="hover"==r?"mouseleave":"focusout",this.$element.on(("hover"==r?"mouseenter":"focusin")+"."+this.type,this.options.selector,d.proxy(this.enter,this)),this.$element.on(o+"."+this.type,this.options.selector,d.proxy(this.leave,this)))}this.options.selector?this._options=d.extend({},this.options,{trigger:"manual",selector:""}):this.fixTitle()},s.prototype.getDefaults=function(){return s.DEFAULTS},s.prototype.getOptions=function(n){var e,t=this.$element.data();for(e in t)t.hasOwnProperty(e)&&-1!==d.inArray(e,l)&&delete t[e];return(n=d.extend({},this.getDefaults(),t,n)).delay&&"number"==typeof n.delay&&(n.delay={show:n.delay,hide:n.delay}),n.sanitize&&(n.template=a(n.template,n.whiteList,n.sanitizeFn)),n},s.prototype.getDelegateOptions=function(){var t={},l=this.getDefaults();return this._options&&d.each(this._options,function(n,e){l[n]!=e&&(t[n]=e)}),t},s.prototype.enter=function(n){var e=n instanceof this.constructor?n:d(n.currentTarget).data("bs."+this.type);if(e||(e=new this.constructor(n.currentTarget,this.getDelegateOptions()),d(n.currentTarget).data("bs."+this.type,e)),n instanceof d.Event&&(e.inState["focusin"==n.type?"focus":"hover"]=!0),e.tip().hasClass("in")||"in"==e.hoverState)e.hoverState="in";else{if(clearTimeout(e.timeout),e.hoverState="in",!e.options.delay||!e.options.delay.show)return e.show();e.timeout=setTimeout(function(){"in"==e.hoverState&&e.show()},e.options.delay.show)}},s.prototype.isInStateTrue=function(){for(var n in this.inState)if(this.inState[n])return!0;return!1},s.prototype.leave=function(n){var e=n instanceof this.constructor?n:d(n.currentTarget).data("bs."+this.type);if(e||(e=new this.constructor(n.currentTarget,this.getDelegateOptions()),d(n.currentTarget).data("bs."+this.type,e)),n instanceof d.Event&&(e.inState["focusout"==n.type?"focus":"hover"]=!1),!e.isInStateTrue()){if(clearTimeout(e.timeout),e.hoverState="out",!e.options.delay||!e.options.delay.hide)return e.hide();e.timeout=setTimeout(function(){"out"==e.hoverState&&e.hide()},e.options.delay.hide)}},s.prototype.show=function(){var e,n,t,l,a,o,r,i=d.Event("show.bs."+this.type);this.hasContent()&&this.enabled&&(this.$element.trigger(i),t=d.contains(this.$element[0].ownerDocument.documentElement,this.$element[0]),!i.isDefaultPrevented()&&t&&(n=(e=this).tip(),o=this.getUID(this.type),this.setContent(),n.attr("id",o),this.$element.attr("aria-describedby",o),this.options.animation&&n.addClass("fade"),r="function"==typeof this.options.placement?this.options.placement.call(this,n[0],this.$element[0]):this.options.placement,(a=(l=/\s?auto?\s?/i).test(r))&&(r=r.replace(l,"")||"top"),n.detach().css({top:0,left:0,display:"block"}).addClass(r).data("bs."+this.type,this),this.options.container?n.appendTo(d(document).find(this.options.container)):n.insertAfter(this.$element),this.$element.trigger("inserted.bs."+this.type),i=this.getPosition(),t=n[0].offsetWidth,o=n[0].offsetHeight,a&&(l=r,a=this.getPosition(this.$viewport),r="bottom"==r&&i.bottom+o>a.bottom?"top":"top"==r&&i.top-oa.width?"left":"left"==r&&i.left-ti.top+i.height&&(a.top=i.top+i.height-o)):(o=e.left-r,t=e.left+r+t,oi.right&&(a.left=i.left+i.width-t)),a},s.prototype.getTitle=function(){var n=this.$element,e=this.options;return n.attr("data-original-title")||("function"==typeof e.title?e.title.call(n[0]):e.title)},s.prototype.getUID=function(n){for(;n+=~~(1e6*Math.random()),document.getElementById(n););return n},s.prototype.tip=function(){if(!this.$tip&&(this.$tip=d(this.options.template),1!=this.$tip.length))throw new Error(this.type+" `template` option must consist of exactly 1 top-level element!");return this.$tip},s.prototype.arrow=function(){return this.$arrow=this.$arrow||this.tip().find(".tooltip-arrow")},s.prototype.enable=function(){this.enabled=!0},s.prototype.disable=function(){this.enabled=!1},s.prototype.toggleEnabled=function(){this.enabled=!this.enabled},s.prototype.toggle=function(n){var e=this;n&&((e=d(n.currentTarget).data("bs."+this.type))||(e=new this.constructor(n.currentTarget,this.getDelegateOptions()),d(n.currentTarget).data("bs."+this.type,e))),n?(e.inState.click=!e.inState.click,e.isInStateTrue()?e.enter(e):e.leave(e)):e.tip().hasClass("in")?e.leave(e):e.enter(e)},s.prototype.destroy=function(){var n=this;clearTimeout(this.timeout),this.hide(function(){n.$element.off("."+n.type).removeData("bs."+n.type),n.$tip&&n.$tip.detach(),n.$tip=null,n.$arrow=null,n.$viewport=null,n.$element=null})},s.prototype.sanitizeHtml=function(n){return a(n,this.options.whiteList,this.options.sanitizeFn)};var e=d.fn.tooltip;d.fn.tooltip=function(l){return this.each(function(){var n=d(this),e=n.data("bs.tooltip"),t="object"==typeof l&&l;!e&&/destroy|hide/.test(l)||(e||n.data("bs.tooltip",e=new s(this,t)),"string"==typeof l&&e[l]())})},d.fn.tooltip.Constructor=s,d.fn.tooltip.noConflict=function(){return d.fn.tooltip=e,this}}(jQuery)},{}],13:[function(n,e,t){!function(l){"use strict";l.fn.emulateTransitionEnd=function(n){var e=!1,t=this;l(this).one("bsTransitionEnd",function(){e=!0});return setTimeout(function(){e||l(t).trigger(l.support.transition.end)},n),this},l(function(){l.support.transition=function(){var n,e=document.createElement("bootstrap"),t={WebkitTransition:"webkitTransitionEnd",MozTransition:"transitionend",OTransition:"oTransitionEnd otransitionend",transition:"transitionend"};for(n in t)if(void 0!==e.style[n])return{end:t[n]};return!1}(),l.support.transition&&(l.event.special.bsTransitionEnd={bindType:l.support.transition.end,delegateType:l.support.transition.end,handle:function(n){if(l(n.target).is(this))return n.handleObj.handler.apply(this,arguments)}})})}(jQuery)},{}],14:[function(n,e,t){"use strict";function l(n){return n&&n.__esModule?n:{default:n}}function a(n){if(n&&n.__esModule)return n;var e={};if(null!=n)for(var t in n)Object.prototype.hasOwnProperty.call(n,t)&&(e[t]=n[t]);return e.default=n,e}t.__esModule=!0;var o=a(n("./handlebars/base")),r=l(n("./handlebars/safe-string")),i=l(n("./handlebars/exception")),s=a(n("./handlebars/utils")),c=a(n("./handlebars/runtime")),u=l(n("./handlebars/no-conflict"));function h(){var e=new o.HandlebarsEnvironment;return s.extend(e,o),e.SafeString=r.default,e.Exception=i.default,e.Utils=s,e.escapeExpression=s.escapeExpression,e.VM=c,e.template=function(n){return c.template(n,e)},e}n=h();n.create=h,u.default(n),n.default=n,t.default=n,e.exports=t.default},{"./handlebars/base":15,"./handlebars/exception":18,"./handlebars/no-conflict":31,"./handlebars/runtime":32,"./handlebars/safe-string":33,"./handlebars/utils":34}],15:[function(n,e,t){"use strict";function l(n){return n&&n.__esModule?n:{default:n}}t.__esModule=!0,t.HandlebarsEnvironment=h;var a=n("./utils"),o=l(n("./exception")),r=n("./helpers"),i=n("./decorators"),s=l(n("./logger")),c=n("./internal/proto-access");t.VERSION="4.7.7";t.COMPILER_REVISION=8;t.LAST_COMPATIBLE_COMPILER_REVISION=7;t.REVISION_CHANGES={1:"<= 1.0.rc.2",2:"== 1.0.0-rc.3",3:"== 1.0.0-rc.4",4:"== 1.x.x",5:"== 2.0.0-alpha.x",6:">= 2.0.0-beta.1",7:">= 4.0.0 <4.3.0",8:">= 4.3.0"};var u="[object Object]";function h(n,e,t){this.helpers=n||{},this.partials=e||{},this.decorators=t||{},r.registerDefaultHelpers(this),i.registerDefaultDecorators(this)}h.prototype={constructor:h,logger:s.default,log:s.default.log,registerHelper:function(n,e){if(a.toString.call(n)===u){if(e)throw new o.default("Arg not supported with multiple helpers");a.extend(this.helpers,n)}else this.helpers[n]=e},unregisterHelper:function(n){delete this.helpers[n]},registerPartial:function(n,e){if(a.toString.call(n)===u)a.extend(this.partials,n);else{if(void 0===e)throw new o.default('Attempting to register a partial called "'+n+'" as undefined');this.partials[n]=e}},unregisterPartial:function(n){delete this.partials[n]},registerDecorator:function(n,e){if(a.toString.call(n)===u){if(e)throw new o.default("Arg not supported with multiple decorators");a.extend(this.decorators,n)}else this.decorators[n]=e},unregisterDecorator:function(n){delete this.decorators[n]},resetLoggedPropertyAccesses:function(){c.resetLoggedProperties()}};n=s.default.log;t.log=n,t.createFrame=a.createFrame,t.logger=s.default},{"./decorators":16,"./exception":18,"./helpers":19,"./internal/proto-access":28,"./logger":30,"./utils":34}],16:[function(n,e,t){"use strict";t.__esModule=!0,t.registerDefaultDecorators=function(n){o.default(n)};var l,a=n("./decorators/inline"),o=(l=a)&&l.__esModule?l:{default:l}},{"./decorators/inline":17}],17:[function(n,e,t){"use strict";t.__esModule=!0;var r=n("../utils");t.default=function(n){n.registerDecorator("inline",function(l,a,o,n){var e=l;return a.partials||(a.partials={},e=function(n,e){var t=o.partials;o.partials=r.extend({},t,a.partials);e=l(n,e);return o.partials=t,e}),a.partials[n.args[0]]=n.fn,e})},e.exports=t.default},{"../utils":34}],18:[function(n,e,t){"use strict";t.__esModule=!0;var s=["description","fileName","lineNumber","endLineNumber","message","name","number","stack"];function c(n,e){var t=e&&e.loc,l=void 0,a=void 0,o=void 0,e=void 0;t&&(l=t.start.line,a=t.end.line,o=t.start.column,e=t.end.column,n+=" - "+l+":"+o);for(var r=Error.prototype.constructor.call(this,n),i=0;i=p.LAST_COMPATIBLE_COMPILER_REVISION&&e<=p.COMPILER_REVISION)){if(e":">",'"':""","'":"'","`":"`","=":"="},a=/[&<>"'`=]/g,o=/[&<>"'`=]/;function r(n){return l[n]}function i(n){for(var e=1;etombstone "+e+"?",function(n){!0===n&&new b({name:e}).tombstoneTopic(t).done(function(){window.location.reload(!0)}).fail(this.handleAJAXError.bind(this))}.bind(this))}});e.exports=n},{"../app_state":36,"../lib/pubsub":41,"../models/channel":43,"../models/node":44,"../models/topic":45,"../router":46,"./base":48,"./channel":50,"./counter":52,"./header":55,"./lookup":57,"./node":59,"./nodes":61,"./topic":64,"./topics":66,bootbox:void 0,bootstrap:1,jquery:void 0}],48:[function(n,e,t){var l=n("jquery"),a=n("underscore"),o=n("backbone"),r=n("../app_state"),i=n("./error.hbs"),n=o.View.extend({constructor:function(n){return this.options=n||{},o.View.prototype.constructor.apply(this,arguments)},initialize:function(){this.subviews=[],this.rendered=!1},template:function(){},skippedRender:function(){},render:function(n){if(this.renderOnce&&this.rendered)return this.skippedRender(),this;this.removeSubviews();var e=this.getRenderCtx(n),n=this.template(e);return this.removed||(this.$el.empty(),this.$el.append(n),this.postRender(e)),this.rendered=!0,this},getRenderCtx:function(n){var e={graph_enabled:r.get("GRAPH_ENABLED"),graph_interval:r.get("graph_interval"),graph_active:r.get("GRAPH_ENABLED")&&"off"!==r.get("graph_interval"),nsqlookupd:r.get("NSQLOOKUPD"),version:r.get("VERSION")};return this.model?e=a.extend(e,this.model.toJSON()):this.collection&&(e=a.extend(e,{collection:this.collection.toJSON()})),e=n?a.extend(e,n):e},postRender:function(){},appendSubview:function(n,e){return this.appendSubviews([n],e)},appendSubviews:function(n,e){this.subviews.push.apply(this.subviews,n),(e?this.$(e):this.$el).append(n.map(function(n){return n.render().delegateEvents().el}))},removeSubviews:function(){for(;this.subviews.length;)this.subviews.pop().remove()},remove:function(){this.removed=!0,this.removeSubviews(),o.View.prototype.remove.apply(this,arguments)},parseErrorMessage:function(n){var e="ERROR: failed to connect to nsqadmin";if(4===n.readyState)try{e=JSON.parse(n.responseText).message}catch(n){e="ERROR: failed to decode JSON - "+n.message}return e},handleAJAXError:function(n){l("#warning, #error").hide(),l("#error .alert").text(this.parseErrorMessage(n)),l("#error").show()},handleViewError:function(n){this.removeSubviews(),this.$el.html(i({message:this.parseErrorMessage(n)}))}});e.exports=n},{"../app_state":36,"./error.hbs":53,backbone:void 0,jquery:void 0,underscore:void 0}],49:[function(n,e,t){n=n("hbsfy/runtime");e.exports=n.template({1:function(n,e,t,l,a){var o=n.lookupProperty||function(n,e){if(Object.prototype.hasOwnProperty.call(n,e))return n[e]};return'
\n
\n
\n

Notice

No producers exist for this topic/channel.\n

See Lookup for more information.\n

\n
\n
\n'},3:function(n,e,t,l,a,o,r){var i,s=null!=e?e:n.nullContext||{},c=n.hooks.helperMissing,u=n.escapeExpression,h=n.lookupProperty||function(n,e){if(Object.prototype.hasOwnProperty.call(n,e))return n[e]};return(null!=(i=h(t,"if").call(s,null!=e?h(e,"isAdmin"):e,{name:"if",hash:{},fn:n.program(4,a,0,o,r),inverse:n.noop,data:a,loc:{start:{line:29,column:0},end:{line:45,column:7}}}))?i:"")+'\n
\n
\n

Channel

\n \n \n \n \n \n'+(null!=(i=h(t,"if").call(s,null!=(i=null!=(i=null!=e?h(e,"e2e_processing_latency"):e)?h(i,"percentiles"):i)?h(i,"length"):i,{name:"if",hash:{},fn:n.program(13,a,0,o,r),inverse:n.noop,data:a,loc:{start:{line:55,column:12},end:{line:57,column:19}}}))?i:"")+" \n \n \n \n \n \n \n \n \n \n "+(null!=(i=h(t,"if").call(s,null!=e?h(e,"graph_active"):e,{name:"if",hash:{},fn:n.program(15,a,0,o,r),inverse:n.noop,data:a,loc:{start:{line:68,column:12},end:{line:68,column:52}}}))?i:"")+"\n \n "+(null!=(i=h(t,"if").call(s,null!=e?h(e,"delivery_msg_count"):e,{name:"if",hash:{},fn:n.program(17,a,0,o,r),inverse:n.noop,data:a,loc:{start:{line:70,column:12},end:{line:70,column:83}}}))?i:"")+"\n"+(null!=(i=h(t,"each").call(s,null!=(i=null!=e?h(e,"e2e_processing_latency"):e)?h(i,"percentiles"):i,{name:"each",hash:{},fn:n.program(19,a,0,o,r),inverse:n.noop,data:a,loc:{start:{line:71,column:12},end:{line:73,column:21}}}))?i:"")+" \n"+(null!=(i=h(t,"each").call(s,null!=e?h(e,"nodes"):e,{name:"each",hash:{},fn:n.program(21,a,0,o,r),inverse:n.noop,data:a,loc:{start:{line:75,column:8},end:{line:142,column:17}}}))?i:"")+' \n \n \n \n \n \n \n \n \n"+(null!=(i=h(t,"if").call(s,null!=e?h(e,"graph_active"):e,{name:"if",hash:{},fn:n.program(28,a,0,o,r),inverse:n.noop,data:a,loc:{start:{line:152,column:12},end:{line:154,column:19}}}))?i:"")+" \n "+(null!=(i=h(t,"if").call(s,null!=e?h(e,"delivery_msg_count"):e,{name:"if",hash:{},fn:n.program(36,a,0,o,r),inverse:n.noop,data:a,loc:{start:{line:156,column:12},end:{line:156,column:54}}}))?i:"")+"\n"+(null!=(i=h(t,"if").call(s,null!=(i=null!=(i=null!=e?h(e,"e2e_processing_latency"):e)?h(i,"percentiles"):i)?h(i,"length"):i,{name:"if",hash:{},fn:n.program(32,a,0,o,r),inverse:n.noop,data:a,loc:{start:{line:157,column:12},end:{line:163,column:19}}}))?i:"")+" \n"+(null!=(i=h(t,"if").call(s,null!=e?h(e,"graph_active"):e,{name:"if",hash:{},fn:n.program(40,a,0,o,r),inverse:n.noop,data:a,loc:{start:{line:165,column:8},end:{line:184,column:15}}}))?i:"")+"
 Message QueuesStatistics
NSQd HostDepthMemory + DiskIn-FlightDeferredRequeuedTimed OutMessagesConnections
Total:'+u((h(t,"commafy")||e&&h(e,"commafy")||c).call(s,null!=e?h(e,"depth"):e,{name:"commafy",hash:{},data:a,loc:{start:{line:145,column:16},end:{line:145,column:33}}}))+""+u((h(t,"commafy")||e&&h(e,"commafy")||c).call(s,null!=e?h(e,"memory_depth"):e,{name:"commafy",hash:{},data:a,loc:{start:{line:146,column:16},end:{line:146,column:40}}}))+" + "+u((h(t,"commafy")||e&&h(e,"commafy")||c).call(s,null!=e?h(e,"backend_depth"):e,{name:"commafy",hash:{},data:a,loc:{start:{line:146,column:43},end:{line:146,column:68}}}))+""+u((h(t,"commafy")||e&&h(e,"commafy")||c).call(s,null!=e?h(e,"in_flight_count"):e,{name:"commafy",hash:{},data:a,loc:{start:{line:147,column:16},end:{line:147,column:43}}}))+""+u((h(t,"commafy")||e&&h(e,"commafy")||c).call(s,null!=e?h(e,"deferred_count"):e,{name:"commafy",hash:{},data:a,loc:{start:{line:148,column:16},end:{line:148,column:42}}}))+""+u((h(t,"commafy")||e&&h(e,"commafy")||c).call(s,null!=e?h(e,"requeue_count"):e,{name:"commafy",hash:{},data:a,loc:{start:{line:149,column:16},end:{line:149,column:41}}}))+""+u((h(t,"commafy")||e&&h(e,"commafy")||c).call(s,null!=e?h(e,"timeout_count"):e,{name:"commafy",hash:{},data:a,loc:{start:{line:150,column:16},end:{line:150,column:41}}}))+""+u((h(t,"commafy")||e&&h(e,"commafy")||c).call(s,null!=e?h(e,"message_count"):e,{name:"commafy",hash:{},data:a,loc:{start:{line:151,column:16},end:{line:151,column:41}}}))+""+u((h(t,"commafy")||e&&h(e,"commafy")||c).call(s,null!=e?h(e,"client_count"):e,{name:"commafy",hash:{},data:a,loc:{start:{line:155,column:16},end:{line:155,column:40}}}))+"
\n
\n
\n"},4:function(n,e,t,l,a){var o=n.lookupProperty||function(n,e){if(Object.prototype.hasOwnProperty.call(n,e))return n[e]};return'
\n
\n \n
\n
\n \n
\n
\n'+(null!=(a=o(t,"if").call(null!=e?e:n.nullContext||{},null!=e?o(e,"paused"):e,{name:"if",hash:{},fn:n.program(5,a,0),inverse:n.program(7,a,0),data:a,loc:{start:{line:38,column:8},end:{line:42,column:15}}}))?a:"")+"
\n
\n"},5:function(n,e,t,l,a){return' \n'},7:function(n,e,t,l,a){return' \n'},9:function(n,e,t,l,a){return"6"},11:function(n,e,t,l,a){return"5"},13:function(n,e,t,l,a){var o=n.lookupProperty||function(n,e){if(Object.prototype.hasOwnProperty.call(n,e))return n[e]};return' E2E Processing Latency\n'},15:function(n,e,t,l,a){return"Rate"},17:function(n,e,t,l,a){return'Delivery'},19:function(n,e,t,l,a){var o=null!=e?e:n.nullContext||{},r=n.hooks.helperMissing,i=n.escapeExpression,n=n.lookupProperty||function(n,e){if(Object.prototype.hasOwnProperty.call(n,e))return n[e]};return" "+i((n(t,"floatToPercent")||e&&n(e,"floatToPercent")||r).call(o,null!=e?n(e,"quantile"):e,{name:"floatToPercent",hash:{},data:a,loc:{start:{line:72,column:20},end:{line:72,column:47}}}))+""+i((n(t,"percSuffix")||e&&n(e,"percSuffix")||r).call(o,null!=e?n(e,"quantile"):e,{name:"percSuffix",hash:{},data:a,loc:{start:{line:72,column:52},end:{line:72,column:75}}}))+"\n"},21:function(n,e,t,l,a,o,r){var i,s=null!=e?e:n.nullContext||{},c=n.hooks.helperMissing,u=n.escapeExpression,h=n.lookupProperty||function(n,e){if(Object.prototype.hasOwnProperty.call(n,e))return n[e]};return" \n \n"+(null!=(i=h(t,"if").call(s,null!=e?h(e,"show_broadcast_address"):e,{name:"if",hash:{},fn:n.program(22,a,0,o,r),inverse:n.program(24,a,0,o,r),data:a,loc:{start:{line:78,column:16},end:{line:82,column:23}}}))?i:"")+" "+(null!=(i=h(t,"if").call(s,null!=e?h(e,"paused"):e,{name:"if",hash:{},fn:n.program(26,a,0,o,r),inverse:n.noop,data:a,loc:{start:{line:83,column:16},end:{line:83,column:85}}}))?i:"")+"\n \n "+u((h(t,"commafy")||e&&h(e,"commafy")||c).call(s,null!=e?h(e,"depth"):e,{name:"commafy",hash:{},data:a,loc:{start:{line:85,column:16},end:{line:85,column:33}}}))+"\n "+u((h(t,"commafy")||e&&h(e,"commafy")||c).call(s,null!=e?h(e,"memory_depth"):e,{name:"commafy",hash:{},data:a,loc:{start:{line:86,column:16},end:{line:86,column:40}}}))+" + "+u((h(t,"commafy")||e&&h(e,"commafy")||c).call(s,null!=e?h(e,"backend_depth"):e,{name:"commafy",hash:{},data:a,loc:{start:{line:86,column:43},end:{line:86,column:68}}}))+"\n "+u((h(t,"commafy")||e&&h(e,"commafy")||c).call(s,null!=e?h(e,"in_flight_count"):e,{name:"commafy",hash:{},data:a,loc:{start:{line:87,column:16},end:{line:87,column:43}}}))+"\n "+u((h(t,"commafy")||e&&h(e,"commafy")||c).call(s,null!=e?h(e,"deferred_count"):e,{name:"commafy",hash:{},data:a,loc:{start:{line:88,column:16},end:{line:88,column:42}}}))+"\n "+u((h(t,"commafy")||e&&h(e,"commafy")||c).call(s,null!=e?h(e,"requeue_count"):e,{name:"commafy",hash:{},data:a,loc:{start:{line:89,column:16},end:{line:89,column:41}}}))+"\n "+u((h(t,"commafy")||e&&h(e,"commafy")||c).call(s,null!=e?h(e,"timeout_count"):e,{name:"commafy",hash:{},data:a,loc:{start:{line:90,column:16},end:{line:90,column:41}}}))+"\n "+u((h(t,"commafy")||e&&h(e,"commafy")||c).call(s,null!=e?h(e,"message_count"):e,{name:"commafy",hash:{},data:a,loc:{start:{line:91,column:16},end:{line:91,column:41}}}))+"\n"+(null!=(i=h(t,"if").call(s,null!=r[1]?h(r[1],"graph_active"):r[1],{name:"if",hash:{},fn:n.program(28,a,0,o,r),inverse:n.noop,data:a,loc:{start:{line:92,column:12},end:{line:94,column:19}}}))?i:"")+" "+u((h(t,"commafy")||e&&h(e,"commafy")||c).call(s,null!=e?h(e,"client_count"):e,{name:"commafy",hash:{},data:a,loc:{start:{line:95,column:16},end:{line:95,column:40}}}))+"\n"+(null!=(i=h(t,"if").call(s,null!=e?h(e,"delivery_msg_count"):e,{name:"if",hash:{},fn:n.program(30,a,0,o,r),inverse:n.noop,data:a,loc:{start:{line:96,column:12},end:{line:113,column:19}}}))?i:"")+(null!=(i=h(t,"if").call(s,null!=(i=null!=(i=null!=e?h(e,"e2e_processing_latency"):e)?h(i,"percentiles"):i)?h(i,"length"):i,{name:"if",hash:{},fn:n.program(32,a,0,o,r),inverse:n.noop,data:a,loc:{start:{line:114,column:12},end:{line:120,column:19}}}))?i:"")+" \n"+(null!=(i=h(t,"if").call(s,null!=r[1]?h(r[1],"graph_active"):r[1],{name:"if",hash:{},fn:n.program(35,a,0,o,r),inverse:n.noop,data:a,loc:{start:{line:122,column:8},end:{line:141,column:15}}}))?i:"")},22:function(n,e,t,l,a){var o=null!=e?e:n.nullContext||{},r=n.hooks.helperMissing,i="function",s=n.escapeExpression,c=n.lookupProperty||function(n,e){if(Object.prototype.hasOwnProperty.call(n,e))return n[e]};return" "+s(typeof(n=null!=(n=c(t,"hostname_port")||(null!=e?c(e,"hostname_port"):e))?n:r)==i?n.call(o,{name:"hostname_port",hash:{},data:a,loc:{start:{line:79,column:16},end:{line:79,column:33}}}):n)+' ('+s(typeof(n=null!=(n=c(t,"node")||(null!=e?c(e,"node"):e))?n:r)==i?n.call(o,{name:"node",hash:{},data:a,loc:{start:{line:79,column:89},end:{line:79,column:97}}}):n)+")\n"},24:function(n,e,t,l,a){var o=null!=e?e:n.nullContext||{},r=n.hooks.helperMissing,i=n.escapeExpression,s="function",c=n.lookupProperty||function(n,e){if(Object.prototype.hasOwnProperty.call(n,e))return n[e]};return' '+i(typeof(n=null!=(n=c(t,"hostname_port")||(null!=e?c(e,"hostname_port"):e))?n:r)==s?n.call(o,{name:"hostname_port",hash:{},data:a,loc:{start:{line:81,column:70},end:{line:81,column:87}}}):n)+"\n"},26:function(n,e,t,l,a){return' paused'},28:function(n,e,t,l,a){var o=n.lookupProperty||function(n,e){if(Object.prototype.hasOwnProperty.call(n,e))return n[e]};return' \n'},30:function(n,e,t,l,a){var o=null!=e?e:n.nullContext||{},r=n.hooks.helperMissing,i="function",s=n.escapeExpression,c=n.lookupProperty||function(n,e){if(Object.prototype.hasOwnProperty.call(n,e))return n[e]};return' \n
\n
\n
\n '+s((c(t,"floatToPercent")||e&&c(e,"floatToPercent")||r).call(o,null!=e?c(e,"zone_local_percentage"):e,{name:"floatToPercent",hash:{},data:a,loc:{start:{line:101,column:50},end:{line:101,column:90}}}))+'%\n
\n
\n '+s((c(t,"floatToPercent")||e&&c(e,"floatToPercent")||r).call(o,null!=e?c(e,"region_local_percentage"):e,{name:"floatToPercent",hash:{},data:a,loc:{start:{line:104,column:50},end:{line:104,column:92}}}))+'%\n
\n
\n '+s((c(t,"floatToPercent")||e&&c(e,"floatToPercent")||r).call(o,null!=e?c(e,"global_percentage"):e,{name:"floatToPercent",hash:{},data:a,loc:{start:{line:107,column:50},end:{line:107,column:86}}}))+'%\n
\n
\n '+s((c(t,"floatToDecimalPercent")||e&&c(e,"floatToDecimalPercent")||r).call(o,null!=e?c(e,"zone_local_percentage"):e,{name:"floatToDecimalPercent",hash:{},data:a,loc:{start:{line:110,column:114},end:{line:110,column:161}}}))+'% | '+s((c(t,"floatToDecimalPercent")||e&&c(e,"floatToDecimalPercent")||r).call(o,null!=e?c(e,"region_local_percentage"):e,{name:"floatToDecimalPercent",hash:{},data:a,loc:{start:{line:110,column:224},end:{line:110,column:273}}}))+'% | '+s((c(t,"floatToDecimalPercent")||e&&c(e,"floatToDecimalPercent")||r).call(o,null!=e?c(e,"global_percentage"):e,{name:"floatToDecimalPercent",hash:{},data:a,loc:{start:{line:110,column:312},end:{line:110,column:355}}}))+"%\n
\n \n"},32:function(n,e,t,l,a){var o=n.lookupProperty||function(n,e){if(Object.prototype.hasOwnProperty.call(n,e))return n[e]};return null!=(e=o(t,"each").call(null!=e?e:n.nullContext||{},null!=(e=null!=e?o(e,"e2e_processing_latency"):e)?o(e,"percentiles"):e,{name:"each",hash:{},fn:n.program(33,a,0),inverse:n.noop,data:a,loc:{start:{line:115,column:16},end:{line:119,column:25}}}))?e:""},33:function(n,e,t,l,a){var o=null!=e?e:n.nullContext||{},r=n.hooks.helperMissing,i=n.escapeExpression,n=n.lookupProperty||function(n,e){if(Object.prototype.hasOwnProperty.call(n,e))return n[e]};return' \n '+i((n(t,"nanotohuman")||e&&n(e,"nanotohuman")||r).call(o,null!=e?n(e,"average"):e,{name:"nanotohuman",hash:{},data:a,loc:{start:{line:117,column:116},end:{line:117,column:139}}}))+"\n \n"},35:function(n,e,t,l,a){var o=null!=e?e:n.nullContext||{},r=n.hooks.helperMissing,i=n.escapeExpression,s=n.lookupProperty||function(n,e){if(Object.prototype.hasOwnProperty.call(n,e))return n[e]};return' \n \n \n \n \n \n \n \n \n \n \n '+(null!=(r=s(t,"if").call(o,null!=e?s(e,"delivery_msg_count"):e,{name:"if",hash:{},fn:n.program(36,a,0),inverse:n.noop,data:a,loc:{start:{line:134,column:12},end:{line:134,column:54}}}))?r:"")+"\n"+(null!=(r=s(t,"if").call(o,null!=(r=null!=(r=null!=e?s(e,"e2e_processing_latency"):e)?s(r,"percentiles"):r)?s(r,"length"):r,{name:"if",hash:{},fn:n.program(38,a,0),inverse:n.noop,data:a,loc:{start:{line:135,column:12},end:{line:139,column:19}}}))?r:"")+" \n"},36:function(n,e,t,l,a){return""},38:function(n,e,t,l,a){var o=n.escapeExpression,r=null!=e?e:n.nullContext||{},i=n.hooks.helperMissing,s=n.lookupProperty||function(n,e){if(Object.prototype.hasOwnProperty.call(n,e))return n[e]};return' \n \n \n'},40:function(n,e,t,l,a){var o=null!=e?e:n.nullContext||{},r=n.hooks.helperMissing,i=n.escapeExpression,s=n.lookupProperty||function(n,e){if(Object.prototype.hasOwnProperty.call(n,e))return n[e]};return' \n \n \n \n \n \n \n \n \n \n \n '+(null!=(r=s(t,"if").call(o,null!=e?s(e,"delivery_msg_count"):e,{name:"if",hash:{},fn:n.program(36,a,0),inverse:n.noop,data:a,loc:{start:{line:177,column:12},end:{line:177,column:54}}}))?r:"")+"\n"+(null!=(r=s(t,"if").call(o,null!=(r=null!=(r=null!=e?s(e,"e2e_processing_latency"):e)?s(r,"percentiles"):r)?s(r,"length"):r,{name:"if",hash:{},fn:n.program(41,a,0),inverse:n.noop,data:a,loc:{start:{line:178,column:12},end:{line:182,column:19}}}))?r:"")+" \n"},41:function(n,e,t,l,a){var o=n.escapeExpression,r=null!=e?e:n.nullContext||{},i=n.hooks.helperMissing,s=n.lookupProperty||function(n,e){if(Object.prototype.hasOwnProperty.call(n,e))return n[e]};return' \n \n \n'},43:function(n,e,t,l,a){return'

Notice

No clients connected to this channel
\n'},45:function(n,e,t,l,a){var o=n.lookupProperty||function(n,e){if(Object.prototype.hasOwnProperty.call(n,e))return n[e]};return' \n \n \n \n \n \n \n \n \n \n \n \n \n'+(null!=(a=o(t,"each").call(null!=e?e:n.nullContext||{},null!=e?o(e,"clients"):e,{name:"each",hash:{},fn:n.program(46,a,0),inverse:n.noop,data:a,loc:{start:{line:210,column:12},end:{line:260,column:21}}}))?a:"")+"
Client HostUser-AgentAttributesNSQd HostIn-FlightReady CountFinishedRequeuedMessagesConnected
\n"},46:function(n,e,t,l,a){var o,r,i=null!=e?e:n.nullContext||{},s=n.hooks.helperMissing,c="function",u=n.escapeExpression,h=n.lookupProperty||function(n,e){if(Object.prototype.hasOwnProperty.call(n,e))return n[e]};return" \n '+u(typeof(r=null!=(r=h(t,"hostname_port")||(null!=e?h(e,"hostname_port"):e))?r:s)==c?r.call(i,{name:"hostname_port",hash:{},data:a,loc:{start:{line:212,column:47},end:{line:212,column:64}}}):r)+(null!=(o=h(t,"if").call(i,null!=e?h(e,"show_client_id"):e,{name:"if",hash:{},fn:n.program(51,a,0),inverse:n.noop,data:a,loc:{start:{line:212,column:64},end:{line:212,column:109}}}))?o:"")+"\n "+(null!=(o=h(t,"if").call(i,null!=(o=null!=e?h(e,"user_agent"):e)?h(o,"length"):o,{name:"if",hash:{},fn:n.program(53,a,0),inverse:n.noop,data:a,loc:{start:{line:213,column:20},end:{line:213,column:81}}}))?o:"")+"\n \n"+(null!=(o=h(t,"if").call(i,null!=e?h(e,"sample_rate"):e,{name:"if",hash:{},fn:n.program(55,a,0),inverse:n.noop,data:a,loc:{start:{line:215,column:20},end:{line:217,column:27}}}))?o:"")+(null!=(o=h(t,"if").call(i,null!=e?h(e,"tls"):e,{name:"if",hash:{},fn:n.program(57,a,0),inverse:n.noop,data:a,loc:{start:{line:218,column:20},end:{line:220,column:27}}}))?o:"")+(null!=(o=h(t,"if").call(i,null!=e?h(e,"deflate"):e,{name:"if",hash:{},fn:n.program(60,a,0),inverse:n.noop,data:a,loc:{start:{line:221,column:20},end:{line:223,column:27}}}))?o:"")+(null!=(o=h(t,"if").call(i,null!=e?h(e,"snappy"):e,{name:"if",hash:{},fn:n.program(62,a,0),inverse:n.noop,data:a,loc:{start:{line:224,column:20},end:{line:226,column:27}}}))?o:"")+(null!=(o=h(t,"if").call(i,null!=e?h(e,"authed"):e,{name:"if",hash:{},fn:n.program(64,a,0),inverse:n.noop,data:a,loc:{start:{line:227,column:20},end:{line:233,column:27}}}))?o:"")+(null!=(o=h(t,"if").call(i,null!=e?h(e,"topology_region"):e,{name:"if",hash:{},fn:n.program(71,a,0),inverse:n.noop,data:a,loc:{start:{line:234,column:20},end:{line:236,column:27}}}))?o:"")+(null!=(o=h(t,"if").call(i,null!=e?h(e,"topology_zone"):e,{name:"if",hash:{},fn:n.program(73,a,0),inverse:n.noop,data:a,loc:{start:{line:237,column:20},end:{line:239,column:27}}}))?o:"")+(null!=(o=h(t,"if").call(i,(h(t,"and")||e&&h(e,"and")||s).call(i,null!=e?h(e,"node_topology_zone"):e,(h(t,"eq")||e&&h(e,"eq")||s).call(i,null!=e?h(e,"topology_zone"):e,null!=e?h(e,"node_topology_zone"):e,{name:"eq",hash:{},data:a,loc:{start:{line:242,column:24},end:{line:242,column:61}}}),{name:"and",hash:{},data:a,loc:{start:{line:240,column:26},end:{line:242,column:62}}}),{name:"if",hash:{},fn:n.program(75,a,0),inverse:n.program(77,a,0),data:a,loc:{start:{line:240,column:20},end:{line:250,column:27}}}))?o:"")+' \n '+u(typeof(r=null!=(r=h(t,"node")||(null!=e?h(e,"node"):e))?r:s)==c?r.call(i,{name:"node",hash:{},data:a,loc:{start:{line:252,column:74},end:{line:252,column:82}}}):r)+"\n "+u((h(t,"commafy")||e&&h(e,"commafy")||s).call(i,null!=e?h(e,"in_flight_count"):e,{name:"commafy",hash:{},data:a,loc:{start:{line:253,column:20},end:{line:253,column:47}}}))+"\n "+u((h(t,"commafy")||e&&h(e,"commafy")||s).call(i,null!=e?h(e,"ready_count"):e,{name:"commafy",hash:{},data:a,loc:{start:{line:254,column:20},end:{line:254,column:43}}}))+"\n "+u((h(t,"commafy")||e&&h(e,"commafy")||s).call(i,null!=e?h(e,"finish_count"):e,{name:"commafy",hash:{},data:a,loc:{start:{line:255,column:20},end:{line:255,column:44}}}))+"\n "+u((h(t,"commafy")||e&&h(e,"commafy")||s).call(i,null!=e?h(e,"requeue_count"):e,{name:"commafy",hash:{},data:a,loc:{start:{line:256,column:20},end:{line:256,column:45}}}))+"\n "+u((h(t,"commafy")||e&&h(e,"commafy")||s).call(i,null!=e?h(e,"message_count"):e,{name:"commafy",hash:{},data:a,loc:{start:{line:257,column:20},end:{line:257,column:45}}}))+"\n "+u((h(t,"nanotohuman")||e&&h(e,"nanotohuman")||s).call(i,null!=e?h(e,"connected"):e,{name:"nanotohuman",hash:{},data:a,loc:{start:{line:258,column:20},end:{line:258,column:45}}}))+"\n \n"},47:function(n,e,t,l,a){return'"bg-zone-local"'},49:function(n,e,t,l,a){return'"bg-region-local"'},51:function(n,e,t,l,a){var o=n.lookupProperty||function(n,e){if(Object.prototype.hasOwnProperty.call(n,e))return n[e]};return" ("+n.escapeExpression("function"==typeof(o=null!=(o=o(t,"client_id")||(null!=e?o(e,"client_id"):e))?o:n.hooks.helperMissing)?o.call(null!=e?e:n.nullContext||{},{name:"client_id",hash:{},data:a,loc:{start:{line:212,column:88},end:{line:212,column:101}}}):o)+")"},53:function(n,e,t,l,a){var o=n.lookupProperty||function(n,e){if(Object.prototype.hasOwnProperty.call(n,e))return n[e]};return""+n.escapeExpression("function"==typeof(o=null!=(o=o(t,"user_agent")||(null!=e?o(e,"user_agent"):e))?o:n.hooks.helperMissing)?o.call(null!=e?e:n.nullContext||{},{name:"user_agent",hash:{},data:a,loc:{start:{line:213,column:52},end:{line:213,column:66}}}):o)+""},55:function(n,e,t,l,a){var o=n.lookupProperty||function(n,e){if(Object.prototype.hasOwnProperty.call(n,e))return n[e]};return' Sampled '+n.escapeExpression("function"==typeof(o=null!=(o=o(t,"sample_rate")||(null!=e?o(e,"sample_rate"):e))?o:n.hooks.helperMissing)?o.call(null!=e?e:n.nullContext||{},{name:"sample_rate",hash:{},data:a,loc:{start:{line:216,column:63},end:{line:216,column:78}}}):o)+"%\n"},57:function(n,e,t,l,a){var o=n.lookupProperty||function(n,e){if(Object.prototype.hasOwnProperty.call(n,e))return n[e]};return' TLS\n"},58:function(n,e,t,l,a){var o=null!=e?e:n.nullContext||{},r=n.hooks.helperMissing,i="function",s=n.escapeExpression,c=n.lookupProperty||function(n,e){if(Object.prototype.hasOwnProperty.call(n,e))return n[e]};return'title="'+s(typeof(n=null!=(n=c(t,"tls_version")||(null!=e?c(e,"tls_version"):e))?n:r)==i?n.call(o,{name:"tls_version",hash:{},data:a,loc:{start:{line:219,column:84},end:{line:219,column:99}}}):n)+" "+s(typeof(n=null!=(n=c(t,"tls_cipher_suite")||(null!=e?c(e,"tls_cipher_suite"):e))?n:r)==i?n.call(o,{name:"tls_cipher_suite",hash:{},data:a,loc:{start:{line:219,column:100},end:{line:219,column:120}}}):n)+" "+s(typeof(n=null!=(n=c(t,"tls_negotiated_protocol")||(null!=e?c(e,"tls_negotiated_protocol"):e))?n:r)==i?n.call(o,{name:"tls_negotiated_protocol",hash:{},data:a,loc:{start:{line:219,column:121},end:{line:219,column:148}}}):n)+" mutual:"+s(typeof(n=null!=(n=c(t,"tls_negotiated_protocol_is_mutual")||(null!=e?c(e,"tls_negotiated_protocol_is_mutual"):e))?n:r)==i?n.call(o,{name:"tls_negotiated_protocol_is_mutual",hash:{},data:a,loc:{start:{line:219,column:156},end:{line:219,column:193}}}):n)+'"'},60:function(n,e,t,l,a){return' Deflate\n'},62:function(n,e,t,l,a){return' Snappy\n'},64:function(n,e,t,l,a){var o,r=null!=e?e:n.nullContext||{},i=n.lookupProperty||function(n,e){if(Object.prototype.hasOwnProperty.call(n,e))return n[e]};return' \n '+(null!=(o=i(t,"if").call(r,null!=e?i(e,"auth_identity_url"):e,{name:"if",hash:{},fn:n.program(65,a,0),inverse:n.noop,data:a,loc:{start:{line:229,column:24},end:{line:229,column:88}}}))?o:"")+'\n \n '+(null!=(o=i(t,"if").call(r,null!=e?i(e,"auth_identity_url"):e,{name:"if",hash:{},fn:n.program(69,a,0),inverse:n.noop,data:a,loc:{start:{line:231,column:24},end:{line:231,column:60}}}))?o:"")+"\n \n"},65:function(n,e,t,l,a){var o=n.lookupProperty||function(n,e){if(Object.prototype.hasOwnProperty.call(n,e))return n[e]};return''},67:function(n,e,t,l,a){var o=n.lookupProperty||function(n,e){if(Object.prototype.hasOwnProperty.call(n,e))return n[e]};return" Identity:"+n.escapeExpression("function"==typeof(o=null!=(o=o(t,"auth_identity")||(null!=e?o(e,"auth_identity"):e))?o:n.hooks.helperMissing)?o.call(null!=e?e:n.nullContext||{},{name:"auth_identity",hash:{},data:a,loc:{start:{line:230,column:113},end:{line:230,column:130}}}):o)},69:function(n,e,t,l,a){return""},71:function(n,e,t,l,a){var o=n.lookupProperty||function(n,e){if(Object.prototype.hasOwnProperty.call(n,e))return n[e]};return' '+n.escapeExpression("function"==typeof(o=null!=(o=o(t,"topology_region")||(null!=e?o(e,"topology_region"):e))?o:n.hooks.helperMissing)?o.call(null!=e?e:n.nullContext||{},{name:"topology_region",hash:{},data:a,loc:{start:{line:235,column:58},end:{line:235,column:77}}}):o)+"\n"},73:function(n,e,t,l,a){var o=n.lookupProperty||function(n,e){if(Object.prototype.hasOwnProperty.call(n,e))return n[e]};return' '+n.escapeExpression("function"==typeof(o=null!=(o=o(t,"topology_zone")||(null!=e?o(e,"topology_zone"):e))?o:n.hooks.helperMissing)?o.call(null!=e?e:n.nullContext||{},{name:"topology_zone",hash:{},data:a,loc:{start:{line:238,column:58},end:{line:238,column:75}}}):o)+"\n"},75:function(n,e,t,l,a){return' zoneLocal\n'},77:function(n,e,t,l,a){var o=null!=e?e:n.nullContext||{},r=n.hooks.helperMissing,i=n.lookupProperty||function(n,e){if(Object.prototype.hasOwnProperty.call(n,e))return n[e]};return null!=(a=i(t,"if").call(o,(i(t,"and")||e&&i(e,"and")||r).call(o,null!=e?i(e,"node_topology_region"):e,(i(t,"eq")||e&&i(e,"eq")||r).call(o,null!=e?i(e,"topology_region"):e,null!=e?i(e,"node_topology_region"):e,{name:"eq",hash:{},data:a,loc:{start:{line:247,column:28},end:{line:247,column:69}}}),{name:"and",hash:{},data:a,loc:{start:{line:245,column:30},end:{line:247,column:70}}}),{name:"if",hash:{},fn:n.program(78,a,0),inverse:n.noop,data:a,loc:{start:{line:245,column:24},end:{line:249,column:31}}}))?a:""},78:function(n,e,t,l,a){return' regionLocal\n'},compiler:[8,">= 4.3.0"],main:function(n,e,t,l,a,o,r){var i,s=null!=e?e:n.nullContext||{},c=n.hooks.helperMissing,u=n.escapeExpression,h="function",p=n.lookupProperty||function(n,e){if(Object.prototype.hasOwnProperty.call(n,e))return n[e]};return(null!=(i=n.invokePartial(p(l,"warning"),e,{name:"warning",data:a,helpers:t,partials:l,decorators:n.decorators}))?i:"")+(null!=(i=n.invokePartial(p(l,"error"),e,{name:"error",data:a,helpers:t,partials:l,decorators:n.decorators}))?i:"")+'\n
    \n
  1. Streams\n
  2. '+u(typeof(l=null!=(l=p(t,"topic")||(null!=e?p(e,"topic"):e))?l:c)==h?l.call(s,{name:"topic",hash:{},data:a,loc:{start:{line:6,column:72},end:{line:6,column:81}}}):l)+'\n
  3. '+u(typeof(l=null!=(l=p(t,"name")||(null!=e?p(e,"name"):e))?l:c)==h?l.call(s,{name:"name",hash:{},data:a,loc:{start:{line:7,column:21},end:{line:7,column:29}}}):l)+'
    4. \n
\n\n
\n
\n
\n

Topic: '+u(typeof(l=null!=(l=p(t,"topic")||(null!=e?p(e,"topic"):e))?l:c)==h?l.call(s,{name:"topic",hash:{},data:a,loc:{start:{line:13,column:30},end:{line:13,column:39}}}):l)+"\n

Channel: "+u(typeof(l=null!=(l=p(t,"name")||(null!=e?p(e,"name"):e))?l:c)==h?l.call(s,{name:"name",hash:{},data:a,loc:{start:{line:14,column:32},end:{line:14,column:40}}}):l)+"\n

\n
\n
\n\n"+(null!=(i=p(t,"unless").call(s,null!=(i=null!=e?p(e,"nodes"):e)?p(i,"length"):i,{name:"unless",hash:{},fn:n.program(1,a,0,o,r),inverse:n.program(3,a,0,o,r),data:a,loc:{start:{line:19,column:0},end:{line:188,column:11}}}))?i:"")+'\n

Client Connections

\n\n
\n
\n'+(null!=(i=p(t,"unless").call(s,null!=(i=null!=e?p(e,"clients"):e)?p(i,"length"):i,{name:"unless",hash:{},fn:n.program(43,a,0,o,r),inverse:n.program(45,a,0,o,r),data:a,loc:{start:{line:194,column:8},end:{line:262,column:19}}}))?i:"")+"
\n
\n"},usePartial:!0,useData:!0,useDepths:!0})},{"hbsfy/runtime":35}],50:[function(t,n,e){var l=t("jquery");window.jQuery=l;t("bootstrap");var a=t("bootbox"),o=t("../lib/pubsub"),r=t("../app_state"),i=t("./base"),s=i.extend({className:"channel container-fluid",template:t("./spinner.hbs"),events:{"click .channel-actions button":"channelAction","click .popup":"showDeliveryBreakdown"},initialize:function(){i.prototype.initialize.apply(this,arguments),this.listenTo(r,"change:graph_interval",this.render);var e=this.model.get("isAdmin");this.model.fetch().done(function(n){this.template=t("./channel.hbs"),this.render({message:n.message,isAdmin:e})}.bind(this)).fail(this.handleViewError.bind(this)).always(o.trigger.bind(o,"view:ready"))},showDeliveryBreakdown:function(n){n.preventDefault(),n.stopPropagation(),document.getElementById(l(n.currentTarget).data("id")).classList.toggle("show")},channelAction:function(n){n.preventDefault(),n.stopPropagation();var t=l(n.currentTarget).data("action"),n="Are you sure you want to "+t+" "+this.model.get("topic")+"/"+this.model.get("name")+"?";a.confirm(n,function(n){var e;!0===n&&("delete"===t?(e=this.model.get("topic"),l.ajax(this.model.url(),{method:"DELETE"}).done(function(){window.location=r.basePath("/topics/"+encodeURIComponent(e))}).fail(this.handleAJAXError.bind(this))):l.post(this.model.url(),JSON.stringify({action:t})).done(function(){window.location.reload(!0)}).fail(this.handleAJAXError.bind(this)))}.bind(this))}});n.exports=s},{"../app_state":36,"../lib/pubsub":41,"./base":48,"./channel.hbs":49,"./spinner.hbs":62,bootbox:void 0,bootstrap:1,jquery:void 0}],51:[function(n,e,t){n=n("hbsfy/runtime");e.exports=n.template({1:function(n,e,t,l,a){var o=n.lookupProperty||function(n,e){if(Object.prototype.hasOwnProperty.call(n,e))return n[e]};return' \n'},compiler:[8,">= 4.3.0"],main:function(n,e,t,l,a){var o,r=n.lookupProperty||function(n,e){if(Object.prototype.hasOwnProperty.call(n,e))return n[e]};return"\n\n"+(null!=(o=n.invokePartial(r(l,"warning"),e,{name:"warning",data:a,helpers:t,partials:l,decorators:n.decorators}))?o:"")+(null!=(o=n.invokePartial(r(l,"error"),e,{name:"error",data:a,helpers:t,partials:l,decorators:n.decorators}))?o:"")+'\n
\n
\n
\n

Messages Processed

\n

\n'+(null!=(o=r(t,"if").call(null!=e?e:n.nullContext||{},null!=e?r(e,"graph_active"):e,{name:"if",hash:{},fn:n.program(1,a,0),inverse:n.noop,data:a,loc:{start:{line:205,column:0},end:{line:207,column:7}}}))?o:"")+"
\n"},usePartial:!0,useData:!0})},{"hbsfy/runtime":35}],52:[function(n,e,t){var l=n("underscore"),i=n("jquery"),a=n("../app_state"),o=n("./base"),n=o.extend({className:"counter container-fluid",template:n("./counter.hbs"),initialize:function(){o.prototype.initialize.apply(this,arguments),this.listenTo(a,"change:graph_interval",function(){clearTimeout(this.poller),clearTimeout(this.animator),this.render(),this.start()}),this.start()},remove:function(){clearTimeout(this.poller),clearTimeout(this.animator),o.prototype.remove.apply(this,arguments)},start:function(){this.poller=null,this.animator=null,this.delta=0,this.looping=!1,this.targetPollInterval=1e4,this.currentNum=-1,this.lastNum=0,this.interval=100,this.graphUrl=null,this.updateStats()},startLoop:function(n){this.interval=n,this.poller=setTimeout(this.updateStats.bind(this),n)},updateStats:function(){var n;i.get(a.apiPath("/counter")).done(function(n){var e,t;this.removed||(t=l.reduce(n.stats,function(n,e){return n+e.message_count},0),-1===this.currentNum?(this.currentNum=t,this.lastNum=t,this.writeCounts(this.currentNum)):t>this.lastNum&&(e=t-this.lastNum,this.delta=e/(this.interval/1e3)/50,this.lastNum=t,this.animator||this.displayFrame()),(t=this.interval)a?((o=i(l[a])).show(),o.find(".top").text(r),o.find(".bottom").text(r)):i(e).append(''+r+''+r+"\n")}i(".numbers .number").each(function(n,e){n>=t.length&&i(e).hide()})}});e.exports=n},{"../app_state":36,"./base":48,"./counter.hbs":51,jquery:void 0,underscore:void 0}],53:[function(n,e,t){n=n("hbsfy/runtime");e.exports=n.template({1:function(n,e,t,l,a){return'style="display: none;"'},compiler:[8,">= 4.3.0"],main:function(n,e,t,l,a){var o,r=null!=e?e:n.nullContext||{},i=n.lookupProperty||function(n,e){if(Object.prototype.hasOwnProperty.call(n,e))return n[e]};return'
\n
\n
\n '+n.escapeExpression("function"==typeof(e=null!=(e=i(t,"message")||(null!=e?i(e,"message"):e))?e:n.hooks.helperMissing)?e.call(r,{name:"message",hash:{},data:a,loc:{start:{line:4,column:12},end:{line:4,column:23}}}):e)+"\n
\n
\n
\n"},useData:!0})},{"hbsfy/runtime":35}],54:[function(n,e,t){n=n("hbsfy/runtime");e.exports=n.template({1:function(n,e,t,l,a){var o,r=null!=e?e:n.nullContext||{},i=n.lookupProperty||function(n,e){if(Object.prototype.hasOwnProperty.call(n,e))return n[e]};return'
  • \n \n
      \n
    • Graph Timeframe
      • \n'+(null!=(a=i(t,"each").call(r,null!=e?i(e,"graph_intervals"):e,{name:"each",hash:{},fn:n.program(2,a,0),inverse:n.noop,data:a,loc:{start:{line:23,column:20},end:{line:25,column:29}}}))?a:"")+"
    \n
    • \n"},2:function(n,e,t,l,a){return'
  • '+n.escapeExpression(n.lambda(e,e))+"
    • \n"},compiler:[8,">= 4.3.0"],main:function(n,e,t,l,a){var o=null!=e?e:n.nullContext||{},r=n.hooks.helperMissing,i=n.escapeExpression,s=n.lookupProperty||function(n,e){if(Object.prototype.hasOwnProperty.call(n,e))return n[e]};return'\n"},useData:!0})},{"hbsfy/runtime":35}],55:[function(n,e,t){var l=n("underscore"),a=n("jquery"),o=n("../app_state"),r=n("./base"),n=r.extend({className:"header",template:n("./header.hbs"),events:{"click .dropdown-menu li":"onGraphIntervalClick"},initialize:function(){r.prototype.initialize.apply(this,arguments),this.listenTo(o,"change:graph_interval",this.render)},getRenderCtx:function(){return l.extend(r.prototype.getRenderCtx.apply(this,arguments),{graph_intervals:["1h","2h","12h","24h","48h","168h","off"],graph_interval:o.get("graph_interval")})},onReset:function(){this.render(),this.$(".dropdown-toggle").dropdown()},onGraphIntervalClick:function(n){n.stopPropagation(),o.set("graph_interval",a(n.target).text())}});e.exports=n},{"../app_state":36,"./base":48,"./header.hbs":54,jquery:void 0,underscore:void 0}],56:[function(n,e,t){n=n("hbsfy/runtime");e.exports=n.template({1:function(n,e,t,l,a){return'
    \n

    Notice

    nsqadmin is not configured with nsqlookupd hosts\n
    \n'},3:function(n,e,t,l,a,o,r){var i,s=null!=e?e:n.nullContext||{},c=n.lookupProperty||function(n,e){if(Object.prototype.hasOwnProperty.call(n,e))return n[e]};return'
    \n
    \n \n \n \n \n'+(null!=(i=c(t,"each").call(s,null!=e?c(e,"nsqlookupd"):e,{name:"each",hash:{},fn:n.program(4,a,0,o,r),inverse:n.noop,data:a,loc:{start:{line:21,column:12},end:{line:23,column:21}}}))?i:"")+'
    nsqlookupd Host
    \n
    \n
    \n\n
    \n
    \n'+(null!=(i=c(t,"if").call(s,null!=e?c(e,"topics"):e,{name:"if",hash:{},fn:n.program(6,a,0,o,r),inverse:n.program(10,a,0,o,r),data:a,loc:{start:{line:30,column:8},end:{line:50,column:15}}}))?i:"")+"
    \n
    \n\n"+(null!=(i=c(t,"if").call(s,null!=e?c(e,"isAdmin"):e,{name:"if",hash:{},fn:n.program(12,a,0,o,r),inverse:n.noop,data:a,loc:{start:{line:54,column:0},end:{line:72,column:7}}}))?i:"")},4:function(n,e,t,l,a){return" "+n.escapeExpression(n.lambda(e,e))+"\n"},6:function(n,e,t,l,a,o,r){var i=n.lookupProperty||function(n,e){if(Object.prototype.hasOwnProperty.call(n,e))return n[e]};return'
    \n Below is a tree of Topics/Channels that are currently inactive (i.e. not produced on any nsqd in the cluster but are present in the lookup data)\n
    \n
      \n'+(null!=(a=i(t,"each").call(null!=e?e:n.nullContext||{},null!=e?i(e,"topics"):e,{name:"each",hash:{},fn:n.program(7,a,0,o,r),inverse:n.noop,data:a,loc:{start:{line:35,column:12},end:{line:46,column:21}}}))?a:"")+"
    \n"},7:function(n,e,t,l,a,o,r){var i,s=null!=e?e:n.nullContext||{},c=n.hooks.helperMissing,u="function",h=n.escapeExpression,p=n.lookupProperty||function(n,e){if(Object.prototype.hasOwnProperty.call(n,e))return n[e]};return'
  • \n '+h(typeof(i=null!=(i=p(t,"name")||(null!=e?p(e,"name"):e))?i:c)==u?i.call(s,{name:"name",hash:{},data:a,loc:{start:{line:37,column:186},end:{line:37,column:194}}}):i)+"\n
      \n"+(null!=(a=p(t,"each").call(s,null!=e?p(e,"channels"):e,{name:"each",hash:{},fn:n.program(8,a,0,o,r),inverse:n.noop,data:a,loc:{start:{line:39,column:20},end:{line:43,column:29}}}))?a:"")+"
    \n
    • \n"},8:function(n,e,t,l,a,o,r){var i=n.lambda,s=n.escapeExpression,c=n.lookupProperty||function(n,e){if(Object.prototype.hasOwnProperty.call(n,e))return n[e]};return'
  • \n '+s(i(e,e))+"\n
    • \n"},10:function(n,e,t,l,a){return'

    Notice

    No inactive Topics
    \n'},12:function(n,e,t,l,a){return'
    \n
    \n
    \n Create Topic/Channel\n
    \n

    This provides a way to setup a stream hierarchy\n before services are deployed to production.\n

    If Channel Name is empty, just the topic is created.\n

    \n
    \n \n \n
    \n \n
    \n
    \n
    \n'},compiler:[8,">= 4.3.0"],main:function(n,e,t,l,a,o,r){var i,s=n.lookupProperty||function(n,e){if(Object.prototype.hasOwnProperty.call(n,e))return n[e]};return(null!=(i=n.invokePartial(s(l,"warning"),e,{name:"warning",data:a,helpers:t,partials:l,decorators:n.decorators}))?i:"")+(null!=(i=n.invokePartial(s(l,"error"),e,{name:"error",data:a,helpers:t,partials:l,decorators:n.decorators}))?i:"")+'\n
    \n
    \n

    Lookup

    \n
    \n
    \n\n'+(null!=(i=s(t,"unless").call(null!=e?e:n.nullContext||{},null!=(i=null!=e?s(e,"nsqlookupd"):e)?s(i,"length"):i,{name:"unless",hash:{},fn:n.program(1,a,0,o,r),inverse:n.program(3,a,0,o,r),data:a,loc:{start:{line:10,column:0},end:{line:73,column:11}}}))?i:"")},usePartial:!0,useData:!0,useDepths:!0})},{"hbsfy/runtime":35}],57:[function(t,n,e){var l=t("underscore"),a=t("jquery"),o=t("../app_state"),r=t("../lib/pubsub"),i=t("./base"),s=t("../models/topic"),c=t("../models/channel"),u=i.extend({className:"lookup container-fluid",template:t("./spinner.hbs"),events:{"click .hierarchy button":"onCreateTopicChannel","click .delete-topic-link":"onDeleteTopic","click .delete-channel-link":"onDeleteChannel"},initialize:function(){i.prototype.initialize.apply(this,arguments);var e=arguments[0].isAdmin;a.ajax(o.apiPath("/topics?inactive=true")).done(function(n){this.template=t("./lookup.hbs"),this.render({topics:l.map(n.topics,function(n,e){return{name:e,channels:n}}),message:n.message,isAdmin:e})}.bind(this)).fail(this.handleViewError.bind(this)).always(r.trigger.bind(r,"view:ready"))},onCreateTopicChannel:function(n){n.preventDefault(),n.stopPropagation();var e=a(n.target.form.elements.topic).val(),n=a(n.target.form.elements.channel).val();""===e&&""===n||a.post(o.apiPath("/topics"),JSON.stringify({topic:e,channel:n})).done(function(){window.location.reload(!0)}).fail(this.handleAJAXError.bind(this))},onDeleteTopic:function(n){n.preventDefault(),n.stopPropagation(),new s({name:a(n.target).data("topic")}).destroy({dataType:"text"}).done(function(){window.location.reload(!0)}).fail(this.handleAJAXError.bind(this))},onDeleteChannel:function(n){n.preventDefault(),n.stopPropagation(),new c({topic:a(n.target).data("topic"),name:a(n.target).data("channel")}).destroy({dataType:"text"}).done(function(){window.location.reload(!0)}).fail(this.handleAJAXError.bind(this))}});n.exports=u},{"../app_state":36,"../lib/pubsub":41,"../models/channel":43,"../models/topic":45,"./base":48,"./lookup.hbs":56,"./spinner.hbs":62,jquery:void 0,underscore:void 0}],58:[function(n,e,t){n=n("hbsfy/runtime");e.exports=n.template({1:function(n,e,t,l,a){var o=null!=e?e:n.nullContext||{},r=n.hooks.helperMissing,i=n.escapeExpression,n=n.lookupProperty||function(n,e){if(Object.prototype.hasOwnProperty.call(n,e))return n[e]};return'
    \n
    \n \n \n \n \n \n \n \n
    \n \n
    GC Pressure
    \n     		\n \n
    GC Pause Percentiles
    \n     		\n \n
    GC Runs
    \n     		\n \n
    Heap Objects In-Use
    \n
    \n
    \n
    \n'},3:function(n,e,t,l,a){return'
    \n

    Notice

    No topics exist on this node.\n
    \n'},5:function(n,e,t,l,a,o,r){var i,s=n.escapeExpression,c=null!=e?e:n.nullContext||{},u=n.hooks.helperMissing,h=n.lookupProperty||function(n,e){if(Object.prototype.hasOwnProperty.call(n,e))return n[e]};return' \n \n \n \n \n \n \n"+(null!=(i=h(t,"each").call(c,null!=e?h(e,"topics"):e,{name:"each",hash:{},fn:n.program(6,a,0,o,r),inverse:n.noop,data:a,loc:{start:{line:50,column:8},end:{line:175,column:17}}}))?i:"")+"
    '+s(n.lambda(null!=(i=null!=r[1]?h(r[1],"topics"):r[1])?h(i,"length"):i,e))+' Topics'+s((h(t,"commafy")||e&&h(e,"commafy")||u).call(c,null!=e?h(e,"total_messages"):e,{name:"commafy",hash:{},data:a,loc:{start:{line:47,column:24},end:{line:47,column:50}}}))+" Messages"+s((h(t,"commafy")||e&&h(e,"commafy")||u).call(c,null!=e?h(e,"total_clients"):e,{name:"commafy",hash:{},data:a,loc:{start:{line:48,column:24},end:{line:48,column:49}}}))+" Clients
    \n"},6:function(n,e,t,l,a,o,r){var i,s=n.escapeExpression,c=null!=e?e:n.nullContext||{},u=n.hooks.helperMissing,h="function",p=n.lookupProperty||function(n,e){if(Object.prototype.hasOwnProperty.call(n,e))return n[e]};return' \n Topic\n Depth\n Memory + Disk\n \n Messages\n Channels\n \n \n \n '+s(typeof(i=null!=(i=p(t,"topic_name")||(null!=e?p(e,"topic_name"):e))?i:u)==h?i.call(c,{name:"topic_name",hash:{},data:a,loc:{start:{line:61,column:158},end:{line:61,column:172}}}):i)+"\n "+(null!=(i=p(t,"if").call(c,null!=e?p(e,"paused"):e,{name:"if",hash:{},fn:n.program(7,a,0,o,r),inverse:n.noop,data:a,loc:{start:{line:62,column:16},end:{line:62,column:84}}}))?i:"")+"\n \n \n "+(null!=(i=p(t,"if").call(c,null!=r[1]?p(r[1],"graph_active"):r[1],{name:"if",hash:{},fn:n.program(9,a,0,o,r),inverse:n.noop,data:a,loc:{start:{line:65,column:16},end:{line:65,column:183}}}))?i:"")+"\n "+s((p(t,"commafy")||e&&p(e,"commafy")||u).call(c,null!=e?p(e,"depth"):e,{name:"commafy",hash:{},data:a,loc:{start:{line:66,column:16},end:{line:66,column:33}}}))+"\n \n "+s((p(t,"commafy")||e&&p(e,"commafy")||u).call(c,null!=e?p(e,"memory_depth"):e,{name:"commafy",hash:{},data:a,loc:{start:{line:68,column:16},end:{line:68,column:40}}}))+" + "+s((p(t,"commafy")||e&&p(e,"commafy")||u).call(c,null!=e?p(e,"backend_depth"):e,{name:"commafy",hash:{},data:a,loc:{start:{line:68,column:43},end:{line:68,column:68}}}))+'\n \n \n '+(null!=(i=p(t,"if").call(c,null!=r[1]?p(r[1],"graph_active"):r[1],{name:"if",hash:{},fn:n.program(11,a,0,o,r),inverse:n.noop,data:a,loc:{start:{line:71,column:16},end:{line:71,column:199}}}))?i:"")+"\n "+s((p(t,"commafy")||e&&p(e,"commafy")||u).call(c,null!=e?p(e,"message_count"):e,{name:"commafy",hash:{},data:a,loc:{start:{line:72,column:16},end:{line:72,column:41}}}))+"\n \n "+s((p(t,"commafy")||e&&p(e,"commafy")||u).call(c,null!=(i=null!=e?p(e,"channels"):e)?p(i,"length"):i,{name:"commafy",hash:{},data:a,loc:{start:{line:74,column:16},end:{line:74,column:43}}}))+"\n \n"+(null!=(i=p(t,"unless").call(c,null!=(i=null!=e?p(e,"channels"):e)?p(i,"length"):i,{name:"unless",hash:{},fn:n.program(13,a,0,o,r),inverse:n.program(15,a,0,o,r),data:a,loc:{start:{line:76,column:8},end:{line:174,column:19}}}))?i:"")},7:function(n,e,t,l,a){return'paused'},9:function(n,e,t,l,a){var o=null!=e?e:n.nullContext||{},r=n.hooks.helperMissing,i=n.escapeExpression,n=n.lookupProperty||function(n,e){if(Object.prototype.hasOwnProperty.call(n,e))return n[e]};return''},11:function(n,e,t,l,a){var o=null!=e?e:n.nullContext||{},r=n.hooks.helperMissing,i=n.escapeExpression,n=n.lookupProperty||function(n,e){if(Object.prototype.hasOwnProperty.call(n,e))return n[e]};return''},13:function(n,e,t,l,a){return' \n \n

    Notice

    No channels exist for this topic.
    \n \n \n'},15:function(n,e,t,l,a,o,r){var i=n.lookupProperty||function(n,e){if(Object.prototype.hasOwnProperty.call(n,e))return n[e]};return null!=(a=i(t,"each").call(null!=e?e:n.nullContext||{},null!=e?i(e,"channels"):e,{name:"each",hash:{},fn:n.program(16,a,0,o,r),inverse:n.noop,data:a,loc:{start:{line:83,column:8},end:{line:173,column:17}}}))?a:""},16:function(n,e,t,l,a,o,r){var i,s=null!=e?e:n.nullContext||{},c=n.hooks.helperMissing,u=n.escapeExpression,h=n.lookupProperty||function(n,e){if(Object.prototype.hasOwnProperty.call(n,e))return n[e]};return' \n \n Channel\n Depth\n Memory + Disk\n In-Flight\n Deferred\n Requeued\n Timed Out\n Messages\n Connections\n \n \n \n \n '+u("function"==typeof(i=null!=(i=h(t,"channel_name")||(null!=e?h(e,"channel_name"):e))?i:c)?i.call(s,{name:"channel_name",hash:{},data:a,loc:{start:{line:99,column:16},end:{line:99,column:32}}}):i)+"\n "+(null!=(i=h(t,"if").call(s,null!=e?h(e,"paused"):e,{name:"if",hash:{},fn:n.program(7,a,0,o,r),inverse:n.noop,data:a,loc:{start:{line:100,column:16},end:{line:100,column:84}}}))?i:"")+"\n \n \n "+(null!=(i=h(t,"if").call(s,null!=r[3]?h(r[3],"graph_active"):r[3],{name:"if",hash:{},fn:n.program(17,a,0,o,r),inverse:n.noop,data:a,loc:{start:{line:103,column:16},end:{line:103,column:225}}}))?i:"")+"\n "+u((h(t,"commafy")||e&&h(e,"commafy")||c).call(s,null!=e?h(e,"depth"):e,{name:"commafy",hash:{},data:a,loc:{start:{line:104,column:16},end:{line:104,column:33}}}))+"\n \n "+u((h(t,"commafy")||e&&h(e,"commafy")||c).call(s,null!=e?h(e,"memory_depth"):e,{name:"commafy",hash:{},data:a,loc:{start:{line:106,column:16},end:{line:106,column:40}}}))+" + "+u((h(t,"commafy")||e&&h(e,"commafy")||c).call(s,null!=e?h(e,"backend_depth"):e,{name:"commafy",hash:{},data:a,loc:{start:{line:106,column:43},end:{line:106,column:68}}}))+"\n "+u((h(t,"commafy")||e&&h(e,"commafy")||c).call(s,null!=e?h(e,"in_flight_count"):e,{name:"commafy",hash:{},data:a,loc:{start:{line:107,column:16},end:{line:107,column:43}}}))+"\n "+u((h(t,"commafy")||e&&h(e,"commafy")||c).call(s,null!=e?h(e,"deferred_count"):e,{name:"commafy",hash:{},data:a,loc:{start:{line:108,column:16},end:{line:108,column:42}}}))+"\n "+u((h(t,"commafy")||e&&h(e,"commafy")||c).call(s,null!=e?h(e,"requeue_count"):e,{name:"commafy",hash:{},data:a,loc:{start:{line:109,column:16},end:{line:109,column:41}}}))+"\n "+u((h(t,"commafy")||e&&h(e,"commafy")||c).call(s,null!=e?h(e,"timeout_count"):e,{name:"commafy",hash:{},data:a,loc:{start:{line:110,column:16},end:{line:110,column:41}}}))+"\n "+u((h(t,"commafy")||e&&h(e,"commafy")||c).call(s,null!=e?h(e,"message_count"):e,{name:"commafy",hash:{},data:a,loc:{start:{line:111,column:16},end:{line:111,column:41}}}))+"\n \n "+(null!=(i=h(t,"if").call(s,null!=r[3]?h(r[3],"graph_active"):r[3],{name:"if",hash:{},fn:n.program(19,a,0,o,r),inverse:n.noop,data:a,loc:{start:{line:113,column:16},end:{line:113,column:229}}}))?i:"")+"\n "+u((h(t,"commafy")||e&&h(e,"commafy")||c).call(s,null!=(i=null!=e?h(e,"clients"):e)?h(i,"length"):i,{name:"commafy",hash:{},data:a,loc:{start:{line:114,column:16},end:{line:114,column:42}}}))+"\n \n \n"+(null!=(i=h(t,"unless").call(s,null!=(i=null!=e?h(e,"clients"):e)?h(i,"length"):i,{name:"unless",hash:{},fn:n.program(21,a,0,o,r),inverse:n.program(23,a,0,o,r),data:a,loc:{start:{line:117,column:8},end:{line:172,column:19}}}))?i:"")},17:function(n,e,t,l,a){var o=null!=e?e:n.nullContext||{},r=n.hooks.helperMissing,i=n.escapeExpression,n=n.lookupProperty||function(n,e){if(Object.prototype.hasOwnProperty.call(n,e))return n[e]};return''},19:function(n,e,t,l,a){var o=null!=e?e:n.nullContext||{},r=n.hooks.helperMissing,i=n.escapeExpression,n=n.lookupProperty||function(n,e){if(Object.prototype.hasOwnProperty.call(n,e))return n[e]};return''},21:function(n,e,t,l,a){return' \n \n

    Notice

    No clients connected to this channel.
    \n \n \n'},23:function(n,e,t,l,a){var o=n.lookupProperty||function(n,e){if(Object.prototype.hasOwnProperty.call(n,e))return n[e]};return" \n \n Client Host\n User-Agent\n \n Attributes\n In-Flight\n Ready Count\n Requeued\n Finished\n Messages\n Connected\n \n"+(null!=(a=o(t,"each").call(null!=e?e:n.nullContext||{},null!=e?o(e,"clients"):e,{name:"each",hash:{},fn:n.program(24,a,0),inverse:n.noop,data:a,loc:{start:{line:137,column:8},end:{line:171,column:17}}}))?a:"")},24:function(n,e,t,l,a){var o,r=null!=e?e:n.nullContext||{},i=n.hooks.helperMissing,s="function",c=n.escapeExpression,u=n.lookupProperty||function(n,e){if(Object.prototype.hasOwnProperty.call(n,e))return n[e]};return' \n \n '+c(typeof(o=null!=(o=u(t,"hostname")||(null!=e?u(e,"hostname"):e))?o:i)==s?o.call(r,{name:"hostname",hash:{},data:a,loc:{start:{line:140,column:43},end:{line:140,column:55}}}):o)+(null!=(o=u(t,"if").call(r,null!=e?u(e,"show_client_id"):e,{name:"if",hash:{},fn:n.program(25,a,0),inverse:n.noop,data:a,loc:{start:{line:140,column:55},end:{line:140,column:100}}}))?o:"")+"\n "+(null!=(o=u(t,"if").call(r,null!=(o=null!=e?u(e,"user_agent"):e)?u(o,"length"):o,{name:"if",hash:{},fn:n.program(27,a,0),inverse:n.noop,data:a,loc:{start:{line:141,column:16},end:{line:141,column:77}}}))?o:"")+"\n \n \n"+(null!=(o=u(t,"if").call(r,null!=e?u(e,"sample_rate"):e,{name:"if",hash:{},fn:n.program(29,a,0),inverse:n.noop,data:a,loc:{start:{line:144,column:16},end:{line:146,column:23}}}))?o:"")+(null!=(o=u(t,"if").call(r,null!=e?u(e,"tls"):e,{name:"if",hash:{},fn:n.program(31,a,0),inverse:n.noop,data:a,loc:{start:{line:147,column:16},end:{line:149,column:23}}}))?o:"")+(null!=(o=u(t,"if").call(r,null!=e?u(e,"deflate"):e,{name:"if",hash:{},fn:n.program(34,a,0),inverse:n.noop,data:a,loc:{start:{line:150,column:16},end:{line:152,column:23}}}))?o:"")+(null!=(o=u(t,"if").call(r,null!=e?u(e,"snappy"):e,{name:"if",hash:{},fn:n.program(36,a,0),inverse:n.noop,data:a,loc:{start:{line:153,column:16},end:{line:155,column:23}}}))?o:"")+(null!=(o=u(t,"if").call(r,null!=e?u(e,"authed"):e,{name:"if",hash:{},fn:n.program(38,a,0),inverse:n.noop,data:a,loc:{start:{line:156,column:16},end:{line:162,column:23}}}))?o:"")+" \n "+c((u(t,"commafy")||e&&u(e,"commafy")||i).call(r,null!=e?u(e,"in_flight_count"):e,{name:"commafy",hash:{},data:a,loc:{start:{line:164,column:16},end:{line:164,column:43}}}))+"\n "+c((u(t,"commafy")||e&&u(e,"commafy")||i).call(r,null!=e?u(e,"ready_count"):e,{name:"commafy",hash:{},data:a,loc:{start:{line:165,column:16},end:{line:165,column:39}}}))+"\n "+c((u(t,"commafy")||e&&u(e,"commafy")||i).call(r,null!=e?u(e,"requeue_count"):e,{name:"commafy",hash:{},data:a,loc:{start:{line:166,column:16},end:{line:166,column:41}}}))+"\n "+c((u(t,"commafy")||e&&u(e,"commafy")||i).call(r,null!=e?u(e,"finish_count"):e,{name:"commafy",hash:{},data:a,loc:{start:{line:167,column:16},end:{line:167,column:40}}}))+"\n "+c((u(t,"commafy")||e&&u(e,"commafy")||i).call(r,null!=e?u(e,"message_count"):e,{name:"commafy",hash:{},data:a,loc:{start:{line:168,column:16},end:{line:168,column:41}}}))+"\n "+c((u(t,"nanotohuman")||e&&u(e,"nanotohuman")||i).call(r,null!=e?u(e,"connected"):e,{name:"nanotohuman",hash:{},data:a,loc:{start:{line:169,column:16},end:{line:169,column:41}}}))+"\n \n"},25:function(n,e,t,l,a){var o=n.lookupProperty||function(n,e){if(Object.prototype.hasOwnProperty.call(n,e))return n[e]};return" ("+n.escapeExpression("function"==typeof(o=null!=(o=o(t,"client_id")||(null!=e?o(e,"client_id"):e))?o:n.hooks.helperMissing)?o.call(null!=e?e:n.nullContext||{},{name:"client_id",hash:{},data:a,loc:{start:{line:140,column:79},end:{line:140,column:92}}}):o)+")"},27:function(n,e,t,l,a){var o=n.lookupProperty||function(n,e){if(Object.prototype.hasOwnProperty.call(n,e))return n[e]};return""+n.escapeExpression("function"==typeof(o=null!=(o=o(t,"user_agent")||(null!=e?o(e,"user_agent"):e))?o:n.hooks.helperMissing)?o.call(null!=e?e:n.nullContext||{},{name:"user_agent",hash:{},data:a,loc:{start:{line:141,column:48},end:{line:141,column:62}}}):o)+""},29:function(n,e,t,l,a){var o=n.lookupProperty||function(n,e){if(Object.prototype.hasOwnProperty.call(n,e))return n[e]};return' Sampled '+n.escapeExpression("function"==typeof(o=null!=(o=o(t,"sample_rate")||(null!=e?o(e,"sample_rate"):e))?o:n.hooks.helperMissing)?o.call(null!=e?e:n.nullContext||{},{name:"sample_rate",hash:{},data:a,loc:{start:{line:145,column:59},end:{line:145,column:74}}}):o)+"%\n"},31:function(n,e,t,l,a){var o=n.lookupProperty||function(n,e){if(Object.prototype.hasOwnProperty.call(n,e))return n[e]};return' TLS\n"},32:function(n,e,t,l,a){var o=null!=e?e:n.nullContext||{},r=n.hooks.helperMissing,i="function",s=n.escapeExpression,c=n.lookupProperty||function(n,e){if(Object.prototype.hasOwnProperty.call(n,e))return n[e]};return'title="'+s(typeof(n=null!=(n=c(t,"tls_version")||(null!=e?c(e,"tls_version"):e))?n:r)==i?n.call(o,{name:"tls_version",hash:{},data:a,loc:{start:{line:148,column:80},end:{line:148,column:95}}}):n)+" "+s(typeof(n=null!=(n=c(t,"tls_cipher_suite")||(null!=e?c(e,"tls_cipher_suite"):e))?n:r)==i?n.call(o,{name:"tls_cipher_suite",hash:{},data:a,loc:{start:{line:148,column:96},end:{line:148,column:116}}}):n)+" "+s(typeof(n=null!=(n=c(t,"tls_negotiated_protocol")||(null!=e?c(e,"tls_negotiated_protocol"):e))?n:r)==i?n.call(o,{name:"tls_negotiated_protocol",hash:{},data:a,loc:{start:{line:148,column:117},end:{line:148,column:144}}}):n)+" mutual:"+s(typeof(n=null!=(n=c(t,"tls_negotiated_protocol_is_mutual")||(null!=e?c(e,"tls_negotiated_protocol_is_mutual"):e))?n:r)==i?n.call(o,{name:"tls_negotiated_protocol_is_mutual",hash:{},data:a,loc:{start:{line:148,column:152},end:{line:148,column:189}}}):n)+'"'},34:function(n,e,t,l,a){return' Deflate\n'},36:function(n,e,t,l,a){return' Snappy\n'},38:function(n,e,t,l,a){var o,r=null!=e?e:n.nullContext||{},i=n.lookupProperty||function(n,e){if(Object.prototype.hasOwnProperty.call(n,e))return n[e]};return' \n '+(null!=(o=i(t,"if").call(r,null!=e?i(e,"auth_identity_url"):e,{name:"if",hash:{},fn:n.program(39,a,0),inverse:n.noop,data:a,loc:{start:{line:158,column:20},end:{line:158,column:84}}}))?o:"")+'\n \n '+(null!=(o=i(t,"if").call(r,null!=e?i(e,"auth_identity_url"):e,{name:"if",hash:{},fn:n.program(43,a,0),inverse:n.noop,data:a,loc:{start:{line:160,column:20},end:{line:160,column:56}}}))?o:"")+"\n \n"},39:function(n,e,t,l,a){var o=n.lookupProperty||function(n,e){if(Object.prototype.hasOwnProperty.call(n,e))return n[e]};return''},41:function(n,e,t,l,a){var o=n.lookupProperty||function(n,e){if(Object.prototype.hasOwnProperty.call(n,e))return n[e]};return" Identity:"+n.escapeExpression("function"==typeof(o=null!=(o=o(t,"auth_identity")||(null!=e?o(e,"auth_identity"):e))?o:n.hooks.helperMissing)?o.call(null!=e?e:n.nullContext||{},{name:"auth_identity",hash:{},data:a,loc:{start:{line:159,column:109},end:{line:159,column:126}}}):o)},43:function(n,e,t,l,a){return""},compiler:[8,">= 4.3.0"],main:function(n,e,t,l,a,o,r){var i,s=null!=e?e:n.nullContext||{},c=n.hooks.helperMissing,u=n.escapeExpression,h=n.lookupProperty||function(n,e){if(Object.prototype.hasOwnProperty.call(n,e))return n[e]};return(null!=(i=n.invokePartial(h(l,"warning"),e,{name:"warning",data:a,helpers:t,partials:l,decorators:n.decorators}))?i:"")+(null!=(i=n.invokePartial(h(l,"error"),e,{name:"error",data:a,helpers:t,partials:l,decorators:n.decorators}))?i:"")+'\n
      \n
    1. Nodes\n
    2. '+u("function"==typeof(u=null!=(u=h(t,"name")||(null!=e?h(e,"name"):e))?u:c)?u.call(s,{name:"name",hash:{},data:a,loc:{start:{line:6,column:23},end:{line:6,column:31}}}):u)+"
      3. \n
    \n\n"+(null!=(i=h(t,"if").call(s,null!=e?h(e,"graph_active"):e,{name:"if",hash:{},fn:n.program(1,a,0,o,r),inverse:n.noop,data:a,loc:{start:{line:9,column:0},end:{line:34,column:7}}}))?i:"")+'\n
    \n
    \n'+(null!=(i=h(t,"unless").call(s,null!=(i=null!=e?h(e,"topics"):e)?h(i,"length"):i,{name:"unless",hash:{},fn:n.program(3,a,0,o,r),inverse:n.program(5,a,0,o,r),data:a,loc:{start:{line:38,column:4},end:{line:177,column:15}}}))?i:"")+"
    \n"},usePartial:!0,useData:!0,useDepths:!0})},{"hbsfy/runtime":35}],59:[function(e,n,t){var l=e("../lib/pubsub"),a=e("../app_state"),o=e("./base"),r=o.extend({className:"node container-fluid",template:e("./spinner.hbs"),initialize:function(){o.prototype.initialize.apply(this,arguments),this.listenTo(a,"change:graph_interval",this.render),this.model.fetch().done(function(n){this.template=e("./node.hbs"),this.render({message:n.message})}.bind(this)).fail(this.handleViewError.bind(this)).always(l.trigger.bind(l,"view:ready"))}});n.exports=r},{"../app_state":36,"../lib/pubsub":41,"./base":48,"./node.hbs":58,"./spinner.hbs":62}],60:[function(n,e,t){n=n("hbsfy/runtime");e.exports=n.template({1:function(n,e,t,l,a){return" Lookupd Conns.\n"},3:function(n,e,t,l,a,o,r){var i,s,c=null!=e?e:n.nullContext||{},u=n.hooks.helperMissing,h="function",p=n.escapeExpression,d=n.lookupProperty||function(n,e){if(Object.prototype.hasOwnProperty.call(n,e))return n[e]};return" \n "+p(typeof(s=null!=(s=d(t,"hostname")||(null!=e?d(e,"hostname"):e))?s:u)==h?s.call(c,{name:"hostname",hash:{},data:a,loc:{start:{line:28,column:20},end:{line:28,column:32}}}):s)+'\n '+p(typeof(s=null!=(s=d(t,"broadcast_address")||(null!=e?d(e,"broadcast_address"):e))?s:u)==h?s.call(c,{name:"broadcast_address",hash:{},data:a,loc:{start:{line:29,column:92},end:{line:29,column:113}}}):s)+"\n "+p(typeof(s=null!=(s=d(t,"tcp_port")||(null!=e?d(e,"tcp_port"):e))?s:u)==h?s.call(c,{name:"tcp_port",hash:{},data:a,loc:{start:{line:30,column:20},end:{line:30,column:32}}}):s)+"\n "+p(typeof(s=null!=(s=d(t,"http_port")||(null!=e?d(e,"http_port"):e))?s:u)==h?s.call(c,{name:"http_port",hash:{},data:a,loc:{start:{line:31,column:20},end:{line:31,column:33}}}):s)+"\n "+p(typeof(s=null!=(s=d(t,"version")||(null!=e?d(e,"version"):e))?s:u)==h?s.call(c,{name:"version",hash:{},data:a,loc:{start:{line:32,column:20},end:{line:32,column:31}}}):s)+"\n "+p(typeof(s=null!=(s=d(t,"topology_region")||(null!=e?d(e,"topology_region"):e))?s:u)==h?s.call(c,{name:"topology_region",hash:{},data:a,loc:{start:{line:33,column:20},end:{line:33,column:39}}}):s)+"\n "+p(typeof(s=null!=(s=d(t,"topology_zone")||(null!=e?d(e,"topology_zone"):e))?s:u)==h?s.call(c,{name:"topology_zone",hash:{},data:a,loc:{start:{line:34,column:20},end:{line:34,column:37}}}):s)+"\n"+(null!=(i=d(t,"if").call(c,null!=(i=null!=r[1]?d(r[1],"nsqlookupd"):r[1])?d(i,"length"):i,{name:"if",hash:{},fn:n.program(6,a,0,o,r),inverse:n.noop,data:a,loc:{start:{line:35,column:16},end:{line:42,column:23}}}))?i:"")+" \n"+(null!=(i=d(t,"if").call(c,null!=(i=null!=e?d(e,"topics"):e)?d(i,"length"):i,{name:"if",hash:{},fn:n.program(11,a,0,o,r),inverse:n.noop,data:a,loc:{start:{line:44,column:16},end:{line:49,column:23}}}))?i:"")+" \n \n"},4:function(n,e,t,l,a){return'class="warning"'},6:function(n,e,t,l,a,o,r){var i,s=null!=e?e:n.nullContext||{},c=n.lookupProperty||function(n,e){if(Object.prototype.hasOwnProperty.call(n,e))return n[e]};return' \n '+n.escapeExpression(n.lambda(null!=(i=null!=e?c(e,"remote_addresses"):e)?c(i,"length"):i,e))+'\n
    \n '+(null!=(i=c(t,"each").call(s,null!=e?c(e,"remote_addresses"):e,{name:"each",hash:{},fn:n.program(9,a,0,o,r),inverse:n.noop,data:a,loc:{start:{line:39,column:24},end:{line:39,column:72}}}))?i:"")+"\n
    \n \n"},7:function(n,e,t,l,a){return"btn-warning"},9:function(n,e,t,l,a){return n.escapeExpression(n.lambda(e,e))+"
    "},11:function(n,e,t,l,a){var o,r=n.lookupProperty||function(n,e){if(Object.prototype.hasOwnProperty.call(n,e))return n[e]};return' '+n.escapeExpression(n.lambda(null!=(o=null!=e?r(e,"topics"):e)?r(o,"length"):o,e))+"\n"+(null!=(o=r(t,"each").call(null!=e?e:n.nullContext||{},null!=e?r(e,"topics"):e,{name:"each",hash:{},fn:n.program(12,a,0),inverse:n.noop,data:a,loc:{start:{line:46,column:20},end:{line:48,column:29}}}))?o:"")},12:function(n,e,t,l,a){var o,r,i=null!=e?e:n.nullContext||{},s=n.hooks.helperMissing,c=n.escapeExpression,u="function",h=n.lookupProperty||function(n,e){if(Object.prototype.hasOwnProperty.call(n,e))return n[e]};return' "+c(typeof(r=null!=(r=h(t,"topic")||(null!=e?h(e,"topic"):e))?r:s)==u?r.call(i,{name:"topic",hash:{},data:a,loc:{start:{line:47,column:223},end:{line:47,column:232}}}):r)+"\n"},13:function(n,e,t,l,a){return"label-warning"},15:function(n,e,t,l,a){return"label-primary"},17:function(n,e,t,l,a){return'title="this topic is currently tombstoned on this node"'},compiler:[8,">= 4.3.0"],main:function(n,e,t,l,a,o,r){var i,s=null!=e?e:n.nullContext||{},c=n.lookupProperty||function(n,e){if(Object.prototype.hasOwnProperty.call(n,e))return n[e]};return(null!=(i=n.invokePartial(c(l,"warning"),e,{name:"warning",data:a,helpers:t,partials:l,decorators:n.decorators}))?i:"")+(null!=(i=n.invokePartial(c(l,"error"),e,{name:"error",data:a,helpers:t,partials:l,decorators:n.decorators}))?i:"")+'\n
    \n
    \n

    NSQd Nodes ('+n.escapeExpression(n.lambda(null!=(i=null!=e?c(e,"collection"):e)?c(i,"length"):i,e))+')

    \n
    \n
    \n\n
    \n
    \n \n \n \n \n \n \n \n \n \n'+(null!=(i=c(t,"if").call(s,null!=(i=null!=e?c(e,"nsqlookupd"):e)?c(i,"length"):i,{name:"if",hash:{},fn:n.program(1,a,0,o,r),inverse:n.noop,data:a,loc:{start:{line:21,column:16},end:{line:23,column:23}}}))?i:"")+" \n \n"+(null!=(i=c(t,"each").call(s,null!=e?c(e,"collection"):e,{name:"each",hash:{},fn:n.program(3,a,0,o,r),inverse:n.noop,data:a,loc:{start:{line:26,column:12},end:{line:52,column:21}}}))?i:"")+"
    HostnameBroadcast AddressTCP PortHTTP PortVersionRegionZoneTopics
    \n
    \n
    \n"},usePartial:!0,useData:!0,useDepths:!0})},{"hbsfy/runtime":35}],61:[function(e,n,t){var l=e("jquery"),a=e("../lib/pubsub"),o=e("../app_state"),r=e("./base"),i=e("../collections/nodes"),s=r.extend({className:"nodes container-fluid",template:e("./spinner.hbs"),events:{"click .conn-count":"onClickConnCount"},initialize:function(){r.prototype.initialize.apply(this,arguments),this.listenTo(o,"change:graph_interval",this.render),this.collection=new i,this.collection.fetch().done(function(n){this.template=e("./nodes.hbs"),this.render({message:n.message})}.bind(this)).fail(this.handleViewError.bind(this)).always(a.trigger.bind(a,"view:ready"))},onClickConnCount:function(n){n.preventDefault(),l(n.target).next().toggle()}});n.exports=s},{"../app_state":36,"../collections/nodes":37,"../lib/pubsub":41,"./base":48,"./nodes.hbs":60,"./spinner.hbs":62,jquery:void 0}],62:[function(n,e,t){n=n("hbsfy/runtime");e.exports=n.template({compiler:[8,">= 4.3.0"],main:function(n,e,t,l,a){return'
    \n \n \n \n
    \n'},useData:!0})},{"hbsfy/runtime":35}],63:[function(n,e,t){n=n("hbsfy/runtime");e.exports=n.template({1:function(n,e,t,l,a){var o=n.lookupProperty||function(n,e){if(Object.prototype.hasOwnProperty.call(n,e))return n[e]};return'
    \n
    \n

    Topic Message Queue

    \n
    \n

    Notice

    No producers exist for this topic.\n

    See Lookup for more information.\n

    \n
    \n
    \n'},3:function(n,e,t,l,a,o,r){var i,s=null!=e?e:n.nullContext||{},c=n.hooks.helperMissing,u=n.escapeExpression,h=n.lookupProperty||function(n,e){if(Object.prototype.hasOwnProperty.call(n,e))return n[e]};return(null!=(i=h(t,"if").call(s,null!=e?h(e,"isAdmin"):e,{name:"if",hash:{},fn:n.program(4,a,0,o,r),inverse:n.noop,data:a,loc:{start:{line:28,column:0},end:{line:44,column:7}}}))?i:"")+'\n
    \n
    \n

    Topic Message Queue

    \n \n'+(null!=(i=h(t,"if").call(s,null!=(i=null!=(i=null!=e?h(e,"e2e_processing_latency"):e)?h(i,"percentiles"):i)?h(i,"length"):i,{name:"if",hash:{},fn:n.program(9,a,0,o,r),inverse:n.noop,data:a,loc:{start:{line:50,column:8},end:{line:55,column:15}}}))?i:"")+" \n \n \n \n \n "+(null!=(i=h(t,"if").call(s,null!=e?h(e,"graph_active"):e,{name:"if",hash:{},fn:n.program(14,a,0,o,r),inverse:n.noop,data:a,loc:{start:{line:61,column:12},end:{line:61,column:52}}}))?i:"")+"\n \n"+(null!=(i=h(t,"each").call(s,null!=(i=null!=e?h(e,"e2e_processing_latency"):e)?h(i,"percentiles"):i,{name:"each",hash:{},fn:n.program(16,a,0,o,r),inverse:n.noop,data:a,loc:{start:{line:63,column:12},end:{line:65,column:21}}}))?i:"")+" \n"+(null!=(i=h(t,"each").call(s,null!=e?h(e,"nodes"):e,{name:"each",hash:{},fn:n.program(18,a,0,o,r),inverse:n.noop,data:a,loc:{start:{line:67,column:8},end:{line:108,column:17}}}))?i:"")+' \n \n \n \n \n "+(null!=(i=h(t,"if").call(s,null!=e?h(e,"graph_active"):e,{name:"if",hash:{},fn:n.program(33,a,0,o,r),inverse:n.noop,data:a,loc:{start:{line:114,column:12},end:{line:114,column:110}}}))?i:"")+"\n \n"+(null!=(i=h(t,"if").call(s,null!=(i=null!=(i=null!=e?h(e,"e2e_processing_latency"):e)?h(i,"percentiles"):i)?h(i,"length"):i,{name:"if",hash:{},fn:n.program(27,a,0,o,r),inverse:n.noop,data:a,loc:{start:{line:116,column:12},end:{line:122,column:19}}}))?i:"")+" \n"+(null!=(i=h(t,"if").call(s,null!=e?h(e,"graph_active"):e,{name:"if",hash:{},fn:n.program(35,a,0,o,r),inverse:n.noop,data:a,loc:{start:{line:124,column:8},end:{line:138,column:15}}}))?i:"")+"
    NSQd HostDepthMemory + DiskMessagesChannels
    Total:'+u((h(t,"commafy")||e&&h(e,"commafy")||c).call(s,null!=e?h(e,"depth"):e,{name:"commafy",hash:{},data:a,loc:{start:{line:111,column:16},end:{line:111,column:33}}}))+""+u((h(t,"commafy")||e&&h(e,"commafy")||c).call(s,null!=e?h(e,"memory_depth"):e,{name:"commafy",hash:{},data:a,loc:{start:{line:112,column:16},end:{line:112,column:40}}}))+" + "+u((h(t,"commafy")||e&&h(e,"commafy")||c).call(s,null!=e?h(e,"backend_depth"):e,{name:"commafy",hash:{},data:a,loc:{start:{line:112,column:43},end:{line:112,column:68}}}))+""+u((h(t,"commafy")||e&&h(e,"commafy")||c).call(s,null!=e?h(e,"message_count"):e,{name:"commafy",hash:{},data:a,loc:{start:{line:113,column:16},end:{line:113,column:41}}}))+""+u((h(t,"commafy")||e&&h(e,"commafy")||c).call(s,null!=(i=null!=e?h(e,"channels"):e)?h(i,"length"):i,{name:"commafy",hash:{},data:a,loc:{start:{line:115,column:16},end:{line:115,column:43}}}))+"
    \n
    \n
    \n"},4:function(n,e,t,l,a){var o=n.lookupProperty||function(n,e){if(Object.prototype.hasOwnProperty.call(n,e))return n[e]};return'
    \n
    \n \n
    \n
    \n \n
    \n
    \n'+(null!=(a=o(t,"if").call(null!=e?e:n.nullContext||{},null!=e?o(e,"paused"):e,{name:"if",hash:{},fn:n.program(5,a,0),inverse:n.program(7,a,0),data:a,loc:{start:{line:37,column:8},end:{line:41,column:15}}}))?a:"")+"
    \n
    \n"},5:function(n,e,t,l,a){return' \n'},7:function(n,e,t,l,a){return' \n'},9:function(n,e,t,l,a){var o=n.lookupProperty||function(n,e){if(Object.prototype.hasOwnProperty.call(n,e))return n[e]};return' \n \n E2E Processing Latency\n \n'},10:function(n,e,t,l,a){return"6"},12:function(n,e,t,l,a){return"5"},14:function(n,e,t,l,a){return"Rate"},16:function(n,e,t,l,a){var o=null!=e?e:n.nullContext||{},r=n.hooks.helperMissing,i=n.escapeExpression,n=n.lookupProperty||function(n,e){if(Object.prototype.hasOwnProperty.call(n,e))return n[e]};return" "+i((n(t,"floatToPercent")||e&&n(e,"floatToPercent")||r).call(o,null!=e?n(e,"quantile"):e,{name:"floatToPercent",hash:{},data:a,loc:{start:{line:64,column:20},end:{line:64,column:47}}}))+""+i((n(t,"percSuffix")||e&&n(e,"percSuffix")||r).call(o,null!=e?n(e,"quantile"):e,{name:"percSuffix",hash:{},data:a,loc:{start:{line:64,column:52},end:{line:64,column:75}}}))+"\n"},18:function(n,e,t,l,a,o,r){var i,s=null!=e?e:n.nullContext||{},c=n.hooks.helperMissing,u=n.escapeExpression,h=n.lookupProperty||function(n,e){if(Object.prototype.hasOwnProperty.call(n,e))return n[e]};return' \n \n \n'+(null!=(i=h(t,"if").call(s,null!=e?h(e,"show_broadcast_address"):e,{name:"if",hash:{},fn:n.program(19,a,0,o,r),inverse:n.program(21,a,0,o,r),data:a,loc:{start:{line:71,column:16},end:{line:75,column:23}}}))?i:"")+" "+(null!=(i=h(t,"if").call(s,null!=e?h(e,"paused"):e,{name:"if",hash:{},fn:n.program(23,a,0,o,r),inverse:n.noop,data:a,loc:{start:{line:76,column:16},end:{line:76,column:85}}}))?i:"")+"\n \n "+u((h(t,"commafy")||e&&h(e,"commafy")||c).call(s,null!=e?h(e,"depth"):e,{name:"commafy",hash:{},data:a,loc:{start:{line:78,column:16},end:{line:78,column:33}}}))+"\n "+u((h(t,"commafy")||e&&h(e,"commafy")||c).call(s,null!=e?h(e,"memory_depth"):e,{name:"commafy",hash:{},data:a,loc:{start:{line:79,column:16},end:{line:79,column:40}}}))+" + "+u((h(t,"commafy")||e&&h(e,"commafy")||c).call(s,null!=e?h(e,"backend_depth"):e,{name:"commafy",hash:{},data:a,loc:{start:{line:79,column:43},end:{line:79,column:68}}}))+"\n "+u((h(t,"commafy")||e&&h(e,"commafy")||c).call(s,null!=e?h(e,"message_count"):e,{name:"commafy",hash:{},data:a,loc:{start:{line:80,column:16},end:{line:80,column:41}}}))+"\n"+(null!=(i=h(t,"if").call(s,null!=r[1]?h(r[1],"graph_active"):r[1],{name:"if",hash:{},fn:n.program(25,a,0,o,r),inverse:n.noop,data:a,loc:{start:{line:81,column:12},end:{line:83,column:19}}}))?i:"")+" "+u((h(t,"commafy")||e&&h(e,"commafy")||c).call(s,null!=(i=null!=e?h(e,"channels"):e)?h(i,"length"):i,{name:"commafy",hash:{},data:a,loc:{start:{line:84,column:16},end:{line:84,column:48}}}))+"\n"+(null!=(i=h(t,"if").call(s,null!=(i=null!=(i=null!=e?h(e,"e2e_processing_latency"):e)?h(i,"percentiles"):i)?h(i,"length"):i,{name:"if",hash:{},fn:n.program(27,a,0,o,r),inverse:n.noop,data:a,loc:{start:{line:85,column:12},end:{line:91,column:19}}}))?i:"")+" \n"+(null!=(i=h(t,"if").call(s,null!=r[1]?h(r[1],"graph_active"):r[1],{name:"if",hash:{},fn:n.program(30,a,0,o,r),inverse:n.noop,data:a,loc:{start:{line:93,column:8},end:{line:107,column:15}}}))?i:"")},19:function(n,e,t,l,a){var o=null!=e?e:n.nullContext||{},r=n.hooks.helperMissing,i="function",s=n.escapeExpression,c=n.lookupProperty||function(n,e){if(Object.prototype.hasOwnProperty.call(n,e))return n[e]};return" "+s(typeof(n=null!=(n=c(t,"hostname_port")||(null!=e?c(e,"hostname_port"):e))?n:r)==i?n.call(o,{name:"hostname_port",hash:{},data:a,loc:{start:{line:72,column:16},end:{line:72,column:33}}}):n)+' ('+s(typeof(n=null!=(n=c(t,"node")||(null!=e?c(e,"node"):e))?n:r)==i?n.call(o,{name:"node",hash:{},data:a,loc:{start:{line:72,column:89},end:{line:72,column:97}}}):n)+")\n"},21:function(n,e,t,l,a){var o=null!=e?e:n.nullContext||{},r=n.hooks.helperMissing,i=n.escapeExpression,s="function",c=n.lookupProperty||function(n,e){if(Object.prototype.hasOwnProperty.call(n,e))return n[e]};return' '+i(typeof(n=null!=(n=c(t,"hostname_port")||(null!=e?c(e,"hostname_port"):e))?n:r)==s?n.call(o,{name:"hostname_port",hash:{},data:a,loc:{start:{line:74,column:70},end:{line:74,column:87}}}):n)+"\n"},23:function(n,e,t,l,a){return' paused'},25:function(n,e,t,l,a){var o=n.lookupProperty||function(n,e){if(Object.prototype.hasOwnProperty.call(n,e))return n[e]};return' \n'},27:function(n,e,t,l,a){var o=n.lookupProperty||function(n,e){if(Object.prototype.hasOwnProperty.call(n,e))return n[e]};return null!=(e=o(t,"each").call(null!=e?e:n.nullContext||{},null!=(e=null!=e?o(e,"e2e_processing_latency"):e)?o(e,"percentiles"):e,{name:"each",hash:{},fn:n.program(28,a,0),inverse:n.noop,data:a,loc:{start:{line:86,column:16},end:{line:90,column:25}}}))?e:""},28:function(n,e,t,l,a){var o=null!=e?e:n.nullContext||{},r=n.hooks.helperMissing,i=n.escapeExpression,n=n.lookupProperty||function(n,e){if(Object.prototype.hasOwnProperty.call(n,e))return n[e]};return' \n '+i((n(t,"nanotohuman")||e&&n(e,"nanotohuman")||r).call(o,null!=e?n(e,"average"):e,{name:"nanotohuman",hash:{},data:a,loc:{start:{line:88,column:116},end:{line:88,column:139}}}))+"\n \n"},30:function(n,e,t,l,a){var o=null!=e?e:n.nullContext||{},r=n.hooks.helperMissing,i=n.escapeExpression,s=n.lookupProperty||function(n,e){if(Object.prototype.hasOwnProperty.call(n,e))return n[e]};return' \n \n \n \n \n \n \n'+(null!=(e=s(t,"if").call(o,null!=(e=null!=(e=null!=e?s(e,"e2e_processing_latency"):e)?s(e,"percentiles"):e)?s(e,"length"):e,{name:"if",hash:{},fn:n.program(31,a,0),inverse:n.noop,data:a,loc:{start:{line:101,column:16},end:{line:105,column:23}}}))?e:"")+" \n"},31:function(n,e,t,l,a){var o=n.escapeExpression,r=null!=e?e:n.nullContext||{},i=n.hooks.helperMissing,s=n.lookupProperty||function(n,e){if(Object.prototype.hasOwnProperty.call(n,e))return n[e]};return' \n \n \n'},33:function(n,e,t,l,a){var o=n.lookupProperty||function(n,e){if(Object.prototype.hasOwnProperty.call(n,e))return n[e]};return''},35:function(n,e,t,l,a){var o=null!=e?e:n.nullContext||{},r=n.hooks.helperMissing,i=n.escapeExpression,s=n.lookupProperty||function(n,e){if(Object.prototype.hasOwnProperty.call(n,e))return n[e]};return' \n \n \n \n \n \n \n'+(null!=(e=s(t,"if").call(o,null!=(e=null!=(e=null!=e?s(e,"e2e_processing_latency"):e)?s(e,"percentiles"):e)?s(e,"length"):e,{name:"if",hash:{},fn:n.program(36,a,0),inverse:n.noop,data:a,loc:{start:{line:132,column:16},end:{line:136,column:23}}}))?e:"")+" \n"},36:function(n,e,t,l,a){var o=n.escapeExpression,r=null!=e?e:n.nullContext||{},i=n.hooks.helperMissing,s=n.lookupProperty||function(n,e){if(Object.prototype.hasOwnProperty.call(n,e))return n[e]};return' \n \n \n'},38:function(n,e,t,l,a){return'
    \n

    Channel Message Queues

    \n
    \n

    Notice

    No channels exist for this topic.\n

    Messages will queue at the topic until a channel is created.\n

    \n'},40:function(n,e,t,l,a,o,r){var i,s=null!=e?e:n.nullContext||{},c=n.lookupProperty||function(n,e){if(Object.prototype.hasOwnProperty.call(n,e))return n[e]};return'
    \n

    Channel Message Queues

    \n \n'+(null!=(i=c(t,"if").call(s,null!=(i=null!=(i=null!=e?c(e,"e2e_processing_latency"):e)?c(i,"percentiles"):i)?c(i,"length"):i,{name:"if",hash:{},fn:n.program(41,a,0,o,r),inverse:n.noop,data:a,loc:{start:{line:157,column:12},end:{line:162,column:19}}}))?i:"")+" \n \n \n \n \n \n \n \n \n \n"+(null!=(i=c(t,"each").call(s,null!=(i=null!=e?c(e,"e2e_processing_latency"):e)?c(i,"percentiles"):i,{name:"each",hash:{},fn:n.program(16,a,0,o,r),inverse:n.noop,data:a,loc:{start:{line:173,column:16},end:{line:175,column:25}}}))?i:"")+" \n\n"+(null!=(i=c(t,"each").call(s,null!=e?c(e,"channels"):e,{name:"each",hash:{},fn:n.program(46,a,0,o,r),inverse:n.noop,data:a,loc:{start:{line:178,column:12},end:{line:218,column:21}}}))?i:"")+"
    ChannelDepthMemory + DiskIn-FlightDeferredRequeuedTimed OutMessagesConnections
    \n"},41:function(n,e,t,l,a){var o=n.lookupProperty||function(n,e){if(Object.prototype.hasOwnProperty.call(n,e))return n[e]};return' \n \n E2E Processing Latency\n \n'},42:function(n,e,t,l,a){return"10"},44:function(n,e,t,l,a){return"9"},46:function(n,e,t,l,a,o,r){var i,s=null!=e?e:n.nullContext||{},c=n.hooks.helperMissing,u=n.escapeExpression,h=n.lookupProperty||function(n,e){if(Object.prototype.hasOwnProperty.call(n,e))return n[e]};return' \n \n '+u("function"==typeof(i=null!=(i=h(t,"channel_name")||(null!=e?h(e,"channel_name"):e))?i:c)?i.call(s,{name:"channel_name",hash:{},data:a,loc:{start:{line:181,column:118},end:{line:181,column:134}}}):i)+"\n "+(null!=(i=h(t,"if").call(s,null!=e?h(e,"paused"):e,{name:"if",hash:{},fn:n.program(47,a,0,o,r),inverse:n.noop,data:a,loc:{start:{line:182,column:20},end:{line:182,column:88}}}))?i:"")+"\n \n "+u((h(t,"commafy")||e&&h(e,"commafy")||c).call(s,null!=e?h(e,"depth"):e,{name:"commafy",hash:{},data:a,loc:{start:{line:184,column:20},end:{line:184,column:37}}}))+"\n "+u((h(t,"commafy")||e&&h(e,"commafy")||c).call(s,null!=e?h(e,"memory_depth"):e,{name:"commafy",hash:{},data:a,loc:{start:{line:185,column:20},end:{line:185,column:44}}}))+" + "+u((h(t,"commafy")||e&&h(e,"commafy")||c).call(s,null!=e?h(e,"backend_depth"):e,{name:"commafy",hash:{},data:a,loc:{start:{line:185,column:47},end:{line:185,column:72}}}))+"\n "+u((h(t,"commafy")||e&&h(e,"commafy")||c).call(s,null!=e?h(e,"in_flight_count"):e,{name:"commafy",hash:{},data:a,loc:{start:{line:186,column:20},end:{line:186,column:47}}}))+"\n "+u((h(t,"commafy")||e&&h(e,"commafy")||c).call(s,null!=e?h(e,"deferred_count"):e,{name:"commafy",hash:{},data:a,loc:{start:{line:187,column:20},end:{line:187,column:46}}}))+"\n "+u((h(t,"commafy")||e&&h(e,"commafy")||c).call(s,null!=e?h(e,"requeue_count"):e,{name:"commafy",hash:{},data:a,loc:{start:{line:188,column:20},end:{line:188,column:45}}}))+"\n "+u((h(t,"commafy")||e&&h(e,"commafy")||c).call(s,null!=e?h(e,"timeout_count"):e,{name:"commafy",hash:{},data:a,loc:{start:{line:189,column:20},end:{line:189,column:45}}}))+"\n "+u((h(t,"commafy")||e&&h(e,"commafy")||c).call(s,null!=e?h(e,"message_count"):e,{name:"commafy",hash:{},data:a,loc:{start:{line:190,column:20},end:{line:190,column:45}}}))+"\n "+u((h(t,"commafy")||e&&h(e,"commafy")||c).call(s,null!=e?h(e,"client_count"):e,{name:"commafy",hash:{},data:a,loc:{start:{line:191,column:20},end:{line:191,column:44}}}))+"\n"+(null!=(i=h(t,"if").call(s,null!=(i=null!=(i=null!=e?h(e,"e2e_processing_latency"):e)?h(i,"percentiles"):i)?h(i,"length"):i,{name:"if",hash:{},fn:n.program(49,a,0,o,r),inverse:n.noop,data:a,loc:{start:{line:192,column:16},end:{line:198,column:23}}}))?i:"")+" \n"+(null!=(i=h(t,"if").call(s,null!=r[1]?h(r[1],"graph_active"):r[1],{name:"if",hash:{},fn:n.program(52,a,0,o,r),inverse:n.noop,data:a,loc:{start:{line:200,column:12},end:{line:217,column:19}}}))?i:"")},47:function(n,e,t,l,a){return'paused'},49:function(n,e,t,l,a){var o=n.lookupProperty||function(n,e){if(Object.prototype.hasOwnProperty.call(n,e))return n[e]};return null!=(e=o(t,"each").call(null!=e?e:n.nullContext||{},null!=(e=null!=e?o(e,"e2e_processing_latency"):e)?o(e,"percentiles"):e,{name:"each",hash:{},fn:n.program(50,a,0),inverse:n.noop,data:a,loc:{start:{line:193,column:20},end:{line:197,column:27}}}))?e:""},50:function(n,e,t,l,a){var o=null!=e?e:n.nullContext||{},r=n.hooks.helperMissing,i=n.escapeExpression,n=n.lookupProperty||function(n,e){if(Object.prototype.hasOwnProperty.call(n,e))return n[e]};return' \n '+i((n(t,"nanotohuman")||e&&n(e,"nanotohuman")||r).call(o,null!=e?n(e,"average"):e,{name:"nanotohuman",hash:{},data:a,loc:{start:{line:195,column:120},end:{line:195,column:143}}}))+"\n \n"},52:function(n,e,t,l,a){var o=null!=e?e:n.nullContext||{},r=n.hooks.helperMissing,i=n.escapeExpression,s=n.lookupProperty||function(n,e){if(Object.prototype.hasOwnProperty.call(n,e))return n[e]};return' \n \n \n \n \n \n \n \n \n \n'+(null!=(e=s(t,"if").call(o,null!=(e=null!=(e=null!=e?s(e,"e2e_processing_latency"):e)?s(e,"percentiles"):e)?s(e,"length"):e,{name:"if",hash:{},fn:n.program(53,a,0),inverse:n.noop,data:a,loc:{start:{line:211,column:16},end:{line:215,column:23}}}))?e:"")+" \n"},53:function(n,e,t,l,a){var o=n.escapeExpression,r=null!=e?e:n.nullContext||{},i=n.hooks.helperMissing,s=n.lookupProperty||function(n,e){if(Object.prototype.hasOwnProperty.call(n,e))return n[e]};return' \n \n \n'},compiler:[8,">= 4.3.0"],main:function(n,e,t,l,a,o,r){var i,s=null!=e?e:n.nullContext||{},c=n.hooks.helperMissing,u=n.escapeExpression,h="function",p=n.lookupProperty||function(n,e){if(Object.prototype.hasOwnProperty.call(n,e))return n[e]};return(null!=(i=n.invokePartial(p(l,"warning"),e,{name:"warning",data:a,helpers:t,partials:l,decorators:n.decorators}))?i:"")+(null!=(i=n.invokePartial(p(l,"error"),e,{name:"error",data:a,helpers:t,partials:l,decorators:n.decorators}))?i:"")+'\n
      \n
    1. Streams\n
    2. '+u(typeof(l=null!=(l=p(t,"name")||(null!=e?p(e,"name"):e))?l:c)==h?l.call(s,{name:"name",hash:{},data:a,loc:{start:{line:6,column:21},end:{line:6,column:29}}}):l)+'
      3. \n
    \n\n
    \n
    \n
    \n

    Topic: '+u(typeof(l=null!=(l=p(t,"name")||(null!=e?p(e,"name"):e))?l:c)==h?l.call(s,{name:"name",hash:{},data:a,loc:{start:{line:12,column:30},end:{line:12,column:38}}}):l)+"\n

    \n
    \n
    \n\n"+(null!=(i=p(t,"unless").call(s,null!=(i=null!=e?p(e,"nodes"):e)?p(i,"length"):i,{name:"unless",hash:{},fn:n.program(1,a,0,o,r),inverse:n.program(3,a,0,o,r),data:a,loc:{start:{line:17,column:0},end:{line:142,column:11}}}))?i:"")+'\n\n
    \n'+(null!=(i=p(t,"unless").call(s,null!=(i=null!=e?p(e,"channels"):e)?p(i,"length"):i,{name:"unless",hash:{},fn:n.program(38,a,0,o,r),inverse:n.program(40,a,0,o,r),data:a,loc:{start:{line:146,column:4},end:{line:220,column:19}}}))?i:"")+"
    \n
    \n"},usePartial:!0,useData:!0,useDepths:!0})},{"hbsfy/runtime":35}],64:[function(t,n,e){var l=t("jquery");window.jQuery=l;t("bootstrap");var a=t("bootbox"),o=t("../lib/pubsub"),r=t("../app_state"),i=t("./base"),s=i.extend({className:"topic container-fluid",template:t("./spinner.hbs"),events:{"click .topic-actions button":"topicAction"},initialize:function(){i.prototype.initialize.apply(this,arguments),this.listenTo(r,"change:graph_interval",this.render);var e=this.model.get("isAdmin");this.model.fetch().done(function(n){this.template=t("./topic.hbs"),this.render({message:n.message,isAdmin:e})}.bind(this)).fail(this.handleViewError.bind(this)).always(o.trigger.bind(o,"view:ready"))},topicAction:function(n){n.preventDefault(),n.stopPropagation();var e=l(n.currentTarget).data("action"),n="Are you sure you want to "+e+" "+this.model.get("name")+"?";a.confirm(n,function(n){!0===n&&("delete"===e?l.ajax(this.model.url(),{method:"DELETE"}).done(function(){window.location=r.basePath("/")}):l.post(this.model.url(),JSON.stringify({action:e})).done(function(){window.location.reload(!0)}).fail(this.handleAJAXError.bind(this)))}.bind(this))}});n.exports=s},{"../app_state":36,"../lib/pubsub":41,"./base":48,"./spinner.hbs":62,"./topic.hbs":63,bootbox:void 0,bootstrap:1,jquery:void 0}],65:[function(n,e,t){n=n("hbsfy/runtime");e.exports=n.template({1:function(n,e,t,l,a,o,r){var i,s=null!=e?e:n.nullContext||{},c=n.lookupProperty||function(n,e){if(Object.prototype.hasOwnProperty.call(n,e))return n[e]};return' \n \n \n '+(null!=(i=c(t,"if").call(s,null!=e?c(e,"graph_active"):e,{name:"if",hash:{},fn:n.program(2,a,0,o,r),inverse:n.noop,data:a,loc:{start:{line:16,column:16},end:{line:16,column:69}}}))?i:"")+"\n "+(null!=(i=c(t,"if").call(s,null!=e?c(e,"graph_active"):e,{name:"if",hash:{},fn:n.program(4,a,0,o,r),inverse:n.noop,data:a,loc:{start:{line:17,column:16},end:{line:17,column:72}}}))?i:"")+"\n "+(null!=(i=c(t,"if").call(s,null!=e?c(e,"graph_active"):e,{name:"if",hash:{},fn:n.program(6,a,0,o,r),inverse:n.noop,data:a,loc:{start:{line:18,column:16},end:{line:18,column:68}}}))?i:"")+"\n \n"+(null!=(i=c(t,"each").call(s,null!=e?c(e,"collection"):e,{name:"each",hash:{},fn:n.program(8,a,0,o,r),inverse:n.noop,data:a,loc:{start:{line:20,column:12},end:{line:27,column:21}}}))?i:"")+"
    Topic
    \n"},2:function(n,e,t,l,a){return'Depth'},4:function(n,e,t,l,a){return'Messages'},6:function(n,e,t,l,a){return'Rate'},8:function(n,e,t,l,a,o,r){var i=null!=e?e:n.nullContext||{},s=n.hooks.helperMissing,c=n.escapeExpression,u=n.lookupProperty||function(n,e){if(Object.prototype.hasOwnProperty.call(n,e))return n[e]};return' \n '+c("function"==typeof(e=null!=(e=u(t,"name")||(null!=e?u(e,"name"):e))?e:s)?e.call(i,{name:"name",hash:{},data:a,loc:{start:{line:22,column:85},end:{line:22,column:93}}}):e)+"\n "+(null!=(e=u(t,"if").call(i,null!=r[1]?u(r[1],"graph_active"):r[1],{name:"if",hash:{},fn:n.program(9,a,0,o,r),inverse:n.noop,data:a,loc:{start:{line:23,column:16},end:{line:23,column:200}}}))?e:"")+"\n "+(null!=(e=u(t,"if").call(i,null!=r[1]?u(r[1],"graph_active"):r[1],{name:"if",hash:{},fn:n.program(11,a,0,o,r),inverse:n.noop,data:a,loc:{start:{line:24,column:16},end:{line:24,column:208}}}))?e:"")+"\n "+(null!=(e=u(t,"if").call(i,null!=r[1]?u(r[1],"graph_active"):r[1],{name:"if",hash:{},fn:n.program(13,a,0,o,r),inverse:n.noop,data:a,loc:{start:{line:25,column:16},end:{line:25,column:111}}}))?e:"")+"\n \n"},9:function(n,e,t,l,a){var o=null!=e?e:n.nullContext||{},r=n.hooks.helperMissing,i=n.escapeExpression,n=n.lookupProperty||function(n,e){if(Object.prototype.hasOwnProperty.call(n,e))return n[e]};return''},11:function(n,e,t,l,a){var o=null!=e?e:n.nullContext||{},r=n.hooks.helperMissing,i=n.escapeExpression,n=n.lookupProperty||function(n,e){if(Object.prototype.hasOwnProperty.call(n,e))return n[e]};return''},13:function(n,e,t,l,a){var o=n.lookupProperty||function(n,e){if(Object.prototype.hasOwnProperty.call(n,e))return n[e]};return''},15:function(n,e,t,l,a){return'

    Notice

    No Topics Found
    \n'},compiler:[8,">= 4.3.0"],main:function(n,e,t,l,a,o,r){var i,s=n.lookupProperty||function(n,e){if(Object.prototype.hasOwnProperty.call(n,e))return n[e]};return(null!=(i=n.invokePartial(s(l,"warning"),e,{name:"warning",data:a,helpers:t,partials:l,decorators:n.decorators}))?i:"")+(null!=(i=n.invokePartial(s(l,"error"),e,{name:"error",data:a,helpers:t,partials:l,decorators:n.decorators}))?i:"")+'\n
    \n
    \n

    Topics

    \n
    \n
    \n\n
    \n
    \n'+(null!=(i=s(t,"if").call(null!=e?e:n.nullContext||{},null!=(i=null!=e?s(e,"collection"):e)?s(i,"length"):i,{name:"if",hash:{},fn:n.program(1,a,0,o,r),inverse:n.program(15,a,0,o,r),data:a,loc:{start:{line:12,column:4},end:{line:31,column:11}}}))?i:"")+"
    \n
    \n"},usePartial:!0,useData:!0,useDepths:!0})},{"hbsfy/runtime":35}],66:[function(e,n,t){var l=e("../lib/pubsub"),a=e("../app_state"),o=e("./base"),r=e("../collections/topics"),i=o.extend({className:"topics container-fluid",template:e("./spinner.hbs"),initialize:function(){o.prototype.initialize.apply(this,arguments),this.listenTo(a,"change:graph_interval",this.render),this.collection=new r,this.collection.fetch().done(function(n){this.template=e("./topics.hbs"),this.render({message:n.message})}.bind(this)).fail(this.handleViewError.bind(this)).always(l.trigger.bind(l,"view:ready"))}});n.exports=i},{"../app_state":36,"../collections/topics":38,"../lib/pubsub":41,"./base":48,"./spinner.hbs":62,"./topics.hbs":65}],67:[function(n,e,t){n=n("hbsfy/runtime");e.exports=n.template({1:function(n,e,t,l,a){return'style="display: none;"'},compiler:[8,">= 4.3.0"],main:function(n,e,t,l,a){var o,r=null!=e?e:n.nullContext||{},i=n.lookupProperty||function(n,e){if(Object.prototype.hasOwnProperty.call(n,e))return n[e]};return'
    \n
    \n
    \n '+n.escapeExpression("function"==typeof(e=null!=(e=i(t,"message")||(null!=e?i(e,"message"):e))?e:n.hooks.helperMissing)?e.call(r,{name:"message",hash:{},data:a,loc:{start:{line:4,column:12},end:{line:4,column:23}}}):e)+"\n
    \n
    \n
    \n"},useData:!0})},{"hbsfy/runtime":35}]},{},[42]); //# sourceMappingURL=main.js.map ================================================ FILE: nsqadmin/static/build/vendor.js ================================================ require=function r(i,o,a){function s(t,e){if(!o[t]){if(!i[t]){var n="function"==typeof require&&require;if(!e&&n)return n(t,!0);if(u)return u(t,!0);throw(n=new Error("Cannot find module '"+t+"'")).code="MODULE_NOT_FOUND",n}n=o[t]={exports:{}},i[t][0].call(n.exports,function(e){return s(i[t][1][e]||e)},n,n.exports,r,i,o,a)}return o[t].exports}for(var u="function"==typeof require&&require,e=0;ethis.length?this.length:r)<0&&(r+=this.length+1);for(var i=[],o=[],a=[],s=[],u={},l=t.add,c=t.merge,f=t.remove,p=!1,d=this.comparator&&null==r&&!1!==t.sort,h=x.isString(this.comparator)?this.comparator:null,g=0;g
    ',header:'
    ',footer:'
    ',closeButton:'',form:'
    ',button:'',option:"",promptMessage:'
    ',inputs:{text:'',textarea:'',email:'',select:'',checkbox:'
    ',radio:'
    ',date:'',time:'',number:'',password:'',range:''}},h={locale:"en",backdrop:"static",animate:!0,className:null,closeButton:!0,show:!0,container:"body",value:"",inputType:"text",swapButtonOrder:!1,centerVertical:!1,multiple:!1,scrollable:!1,reusable:!1};function l(e,t,n){return c.extend(!0,{},e,function(e,t){var n=e.length,r={};if(n<1||2").attr("label",t.group)),n=o[t.group]);var r=c(d.option);r.attr("value",t.value).text(t.text),n.append(r)}),m(o,function(e,t){u.append(t)}),u.val(i.value);break;case"checkbox":var s=c.isArray(i.value)?i.value:[i.value];if(!(a=i.inputOptions||[]).length)throw new Error('prompt with "inputType" set to "checkbox" requires at least one option');u=c('
    '),m(a,function(e,n){if(n.value===f||n.text===f)throw new Error('each option needs a "value" property and a "text" property');var r=c(d.inputs[i.inputType]);r.find("input").attr("value",n.value),r.find("label").append("\n"+n.text),m(s,function(e,t){t===n.value&&r.find("input").prop("checked",!0)}),u.append(r)});break;case"radio":if(i.value!==f&&c.isArray(i.value))throw new Error('prompt with "inputType" set to "radio" requires a single, non-array value for "value"');if(!(a=i.inputOptions||[]).length)throw new Error('prompt with "inputType" set to "radio" requires at least one option');var u=c('
    '),l=!0;m(a,function(e,t){if(t.value===f||t.text===f)throw new Error('each option needs a "value" property and a "text" property');var n=c(d.inputs[i.inputType]);n.find("input").attr("value",t.value),n.find("label").append("\n"+t.text),i.value!==f&&t.value===i.value&&(n.find("input").prop("checked",!0),l=!1),u.append(n)}),l&&u.find('input[type="radio"]').first().prop("checked",!0)}return r.append(u),r.on("submit",function(e){e.preventDefault(),e.stopPropagation(),t.find(".bootbox-accept").trigger("click")}),""!==c.trim(i.message)&&(n=c(d.promptMessage).html(i.message),r.prepend(n)),i.message=r,(t=p.dialog(i)).off("shown.bs.modal",y),t.on("shown.bs.modal",function(){u.focus()}),!0===e&&t.modal("show"),t},p})},{jquery:"jquery"}],jquery:[function(e,n,t){!function(e,t){"use strict";"object"==typeof n&&"object"==typeof n.exports?n.exports=e.document?t(e,!0):function(e){if(!e.document)throw new Error("jQuery requires a window with a document");return t(e)}:t(e)}("undefined"!=typeof window?window:this,function(C,e){"use strict";function g(e){return null!=e&&e===e.window}var t=[],n=Object.getPrototypeOf,s=t.slice,v=t.flat?function(e){return t.flat.call(e)}:function(e){return t.concat.apply([],e)},u=t.push,i=t.indexOf,r={},o=r.toString,m=r.hasOwnProperty,a=m.toString,l=a.call(Object),y={},b=function(e){return"function"==typeof e&&"number"!=typeof e.nodeType&&"function"!=typeof e.item},E=C.document,c={type:!0,src:!0,nonce:!0,noModule:!0};function x(e,t,n){var r,i,o=(n=n||E).createElement("script");if(o.text=e,t)for(r in c)(i=t[r]||t.getAttribute&&t.getAttribute(r))&&o.setAttribute(r,i);n.head.appendChild(o).parentNode.removeChild(o)}function h(e){return null==e?e+"":"object"==typeof e||"function"==typeof e?r[o.call(e)]||"object":typeof e}var T=function(e,t){return new T.fn.init(e,t)};function f(e){var t=!!e&&"length"in e&&e.length,n=h(e);return!b(e)&&!g(e)&&("array"===n||0===t||"number"==typeof t&&0>10|55296,1023&e|56320))}function r(){C()}var e,p,x,o,i,d,h,g,w,u,l,C,E,a,T,v,s,c,m,k="sizzle"+ +new Date,y=n.document,N=0,b=0,A=ue(),O=ue(),S=ue(),j=ue(),_=function(e,t){return e===t&&(l=!0),0},L={}.hasOwnProperty,t=[],I=t.pop,D=t.push,M=t.push,q=t.slice,R=function(e,t){for(var n=0,r=e.length;n+~]|"+P+")"+P+"*"),V=new RegExp(P+"|>"),X=new RegExp(K),Y=new RegExp("^"+F+"$"),J={ID:new RegExp("^#("+F+")"),CLASS:new RegExp("^\\.("+F+")"),TAG:new RegExp("^("+F+"|[*])"),ATTR:new RegExp("^"+B),PSEUDO:new RegExp("^"+K),CHILD:new RegExp("^:(only|first|last|nth|nth-last)-(child|of-type)(?:\\("+P+"*(even|odd|(([+-]|)(\\d*)n|)"+P+"*(?:([+-]|)"+P+"*(\\d+)|))"+P+"*\\)|)","i"),bool:new RegExp("^(?:"+H+")$","i"),needsContext:new RegExp("^"+P+"*[>+~]|:(even|odd|eq|gt|lt|nth|first|last)(?:\\("+P+"*((?:-\\d)?\\d*)"+P+"*\\)|)(?=[^-]|$)","i")},G=/HTML$/i,Q=/^(?:input|select|textarea|button)$/i,Z=/^h\d$/i,ee=/^[^{]+\{\s*\[native \w/,te=/^(?:#([\w-]+)|(\w+)|\.([\w-]+))$/,ne=/[+~]/,re=new RegExp("\\\\[\\da-fA-F]{1,6}"+P+"?|\\\\([^\\r\\n\\f])","g"),ie=/([\0-\x1f\x7f]|^-?\d)|^-$|[^\0-\x1f\x7f-\uFFFF\w-]/g,oe=function(e,t){return t?"\0"===e?"�":e.slice(0,-1)+"\\"+e.charCodeAt(e.length-1).toString(16)+" ":"\\"+e},ae=ye(function(e){return!0===e.disabled&&"fieldset"===e.nodeName.toLowerCase()},{dir:"parentNode",next:"legend"});try{M.apply(t=q.call(y.childNodes),y.childNodes),t[y.childNodes.length].nodeType}catch(e){M={apply:t.length?function(e,t){D.apply(e,q.call(t))}:function(e,t){for(var n=e.length,r=0;e[n++]=t[r++];);e.length=n-1}}}function se(t,e,n,r){var i,o,a,s,u,l,c=e&&e.ownerDocument,f=e?e.nodeType:9;if(n=n||[],"string"!=typeof t||!t||1!==f&&9!==f&&11!==f)return n;if(!r&&(C(e),e=e||E,T)){if(11!==f&&(s=te.exec(t)))if(l=s[1]){if(9===f){if(!(o=e.getElementById(l)))return n;if(o.id===l)return n.push(o),n}else if(c&&(o=c.getElementById(l))&&m(e,o)&&o.id===l)return n.push(o),n}else{if(s[2])return M.apply(n,e.getElementsByTagName(t)),n;if((l=s[3])&&p.getElementsByClassName&&e.getElementsByClassName)return M.apply(n,e.getElementsByClassName(l)),n}if(p.qsa&&!j[t+" "]&&(!v||!v.test(t))&&(1!==f||"object"!==e.nodeName.toLowerCase())){if(l=t,c=e,1===f&&(V.test(t)||U.test(t))){for((c=ne.test(t)&&ge(e.parentNode)||e)===e&&p.scope||((a=e.getAttribute("id"))?a=a.replace(ie,oe):e.setAttribute("id",a=k)),i=(u=d(t)).length;i--;)u[i]=(a?"#"+a:":scope")+" "+me(u[i]);l=u.join(",")}try{return M.apply(n,c.querySelectorAll(l)),n}catch(e){j(t,!0)}finally{a===k&&e.removeAttribute("id")}}}return g(t.replace(W,"$1"),e,n,r)}function ue(){var n=[];function r(e,t){return n.push(e+" ")>x.cacheLength&&delete r[n.shift()],r[e+" "]=t}return r}function le(e){return e[k]=!0,e}function ce(e){var t=E.createElement("fieldset");try{return!!e(t)}catch(e){return!1}finally{t.parentNode&&t.parentNode.removeChild(t),t=null}}function fe(e,t){for(var n=e.split("|"),r=n.length;r--;)x.attrHandle[n[r]]=t}function pe(e,t){var n=t&&e,r=n&&1===e.nodeType&&1===t.nodeType&&e.sourceIndex-t.sourceIndex;if(r)return r;if(n)for(;n=n.nextSibling;)if(n===t)return-1;return e?1:-1}function de(t){return function(e){return"form"in e?e.parentNode&&!1===e.disabled?"label"in e?"label"in e.parentNode?e.parentNode.disabled===t:e.disabled===t:e.isDisabled===t||e.isDisabled!==!t&&ae(e)===t:e.disabled===t:"label"in e&&e.disabled===t}}function he(a){return le(function(o){return o=+o,le(function(e,t){for(var n,r=a([],e.length,o),i=r.length;i--;)e[n=r[i]]&&(e[n]=!(t[n]=e[n]))})})}function ge(e){return e&&void 0!==e.getElementsByTagName&&e}for(e in p=se.support={},i=se.isXML=function(e){var t=e&&e.namespaceURI,e=e&&(e.ownerDocument||e).documentElement;return!G.test(t||e&&e.nodeName||"HTML")},C=se.setDocument=function(e){var t,e=e?e.ownerDocument||e:y;return e!=E&&9===e.nodeType&&e.documentElement&&(a=(E=e).documentElement,T=!i(E),y!=E&&(t=E.defaultView)&&t.top!==t&&(t.addEventListener?t.addEventListener("unload",r,!1):t.attachEvent&&t.attachEvent("onunload",r)),p.scope=ce(function(e){return a.appendChild(e).appendChild(E.createElement("div")),void 0!==e.querySelectorAll&&!e.querySelectorAll(":scope fieldset div").length}),p.attributes=ce(function(e){return e.className="i",!e.getAttribute("className")}),p.getElementsByTagName=ce(function(e){return e.appendChild(E.createComment("")),!e.getElementsByTagName("*").length}),p.getElementsByClassName=ee.test(E.getElementsByClassName),p.getById=ce(function(e){return a.appendChild(e).id=k,!E.getElementsByName||!E.getElementsByName(k).length}),p.getById?(x.filter.ID=function(e){var t=e.replace(re,f);return function(e){return e.getAttribute("id")===t}},x.find.ID=function(e,t){if(void 0!==t.getElementById&&T){e=t.getElementById(e);return e?[e]:[]}}):(x.filter.ID=function(e){var t=e.replace(re,f);return function(e){e=void 0!==e.getAttributeNode&&e.getAttributeNode("id");return e&&e.value===t}},x.find.ID=function(e,t){if(void 0!==t.getElementById&&T){var n,r,i,o=t.getElementById(e);if(o){if((n=o.getAttributeNode("id"))&&n.value===e)return[o];for(i=t.getElementsByName(e),r=0;o=i[r++];)if((n=o.getAttributeNode("id"))&&n.value===e)return[o]}return[]}}),x.find.TAG=p.getElementsByTagName?function(e,t){return void 0!==t.getElementsByTagName?t.getElementsByTagName(e):p.qsa?t.querySelectorAll(e):void 0}:function(e,t){var n,r=[],i=0,o=t.getElementsByTagName(e);if("*"!==e)return o;for(;n=o[i++];)1===n.nodeType&&r.push(n);return r},x.find.CLASS=p.getElementsByClassName&&function(e,t){if(void 0!==t.getElementsByClassName&&T)return t.getElementsByClassName(e)},s=[],v=[],(p.qsa=ee.test(E.querySelectorAll))&&(ce(function(e){var t;a.appendChild(e).innerHTML="",e.querySelectorAll("[msallowcapture^='']").length&&v.push("[*^$]="+P+"*(?:''|\"\")"),e.querySelectorAll("[selected]").length||v.push("\\["+P+"*(?:value|"+H+")"),e.querySelectorAll("[id~="+k+"-]").length||v.push("~="),(t=E.createElement("input")).setAttribute("name",""),e.appendChild(t),e.querySelectorAll("[name='']").length||v.push("\\["+P+"*name"+P+"*="+P+"*(?:''|\"\")"),e.querySelectorAll(":checked").length||v.push(":checked"),e.querySelectorAll("a#"+k+"+*").length||v.push(".#.+[+~]"),e.querySelectorAll("\\\f"),v.push("[\\r\\n\\f]")}),ce(function(e){e.innerHTML="";var t=E.createElement("input");t.setAttribute("type","hidden"),e.appendChild(t).setAttribute("name","D"),e.querySelectorAll("[name=d]").length&&v.push("name"+P+"*[*^$|!~]?="),2!==e.querySelectorAll(":enabled").length&&v.push(":enabled",":disabled"),a.appendChild(e).disabled=!0,2!==e.querySelectorAll(":disabled").length&&v.push(":enabled",":disabled"),e.querySelectorAll("*,:x"),v.push(",.*:")})),(p.matchesSelector=ee.test(c=a.matches||a.webkitMatchesSelector||a.mozMatchesSelector||a.oMatchesSelector||a.msMatchesSelector))&&ce(function(e){p.disconnectedMatch=c.call(e,"*"),c.call(e,"[s!='']:x"),s.push("!=",K)}),v=v.length&&new RegExp(v.join("|")),s=s.length&&new RegExp(s.join("|")),t=ee.test(a.compareDocumentPosition),m=t||ee.test(a.contains)?function(e,t){var n=9===e.nodeType?e.documentElement:e,t=t&&t.parentNode;return e===t||!(!t||1!==t.nodeType||!(n.contains?n.contains(t):e.compareDocumentPosition&&16&e.compareDocumentPosition(t)))}:function(e,t){if(t)for(;t=t.parentNode;)if(t===e)return!0;return!1},_=t?function(e,t){if(e===t)return l=!0,0;var n=!e.compareDocumentPosition-!t.compareDocumentPosition;return n||(1&(n=(e.ownerDocument||e)==(t.ownerDocument||t)?e.compareDocumentPosition(t):1)||!p.sortDetached&&t.compareDocumentPosition(e)===n?e==E||e.ownerDocument==y&&m(y,e)?-1:t==E||t.ownerDocument==y&&m(y,t)?1:u?R(u,e)-R(u,t):0:4&n?-1:1)}:function(e,t){if(e===t)return l=!0,0;var n,r=0,i=e.parentNode,o=t.parentNode,a=[e],s=[t];if(!i||!o)return e==E?-1:t==E?1:i?-1:o?1:u?R(u,e)-R(u,t):0;if(i===o)return pe(e,t);for(n=e;n=n.parentNode;)a.unshift(n);for(n=t;n=n.parentNode;)s.unshift(n);for(;a[r]===s[r];)r++;return r?pe(a[r],s[r]):a[r]==y?-1:s[r]==y?1:0}),E},se.matches=function(e,t){return se(e,null,null,t)},se.matchesSelector=function(e,t){if(C(e),p.matchesSelector&&T&&!j[t+" "]&&(!s||!s.test(t))&&(!v||!v.test(t)))try{var n=c.call(e,t);if(n||p.disconnectedMatch||e.document&&11!==e.document.nodeType)return n}catch(e){j(t,!0)}return 0":{dir:"parentNode",first:!0}," ":{dir:"parentNode"},"+":{dir:"previousSibling",first:!0},"~":{dir:"previousSibling"}},preFilter:{ATTR:function(e){return e[1]=e[1].replace(re,f),e[3]=(e[3]||e[4]||e[5]||"").replace(re,f),"~="===e[2]&&(e[3]=" "+e[3]+" "),e.slice(0,4)},CHILD:function(e){return e[1]=e[1].toLowerCase(),"nth"===e[1].slice(0,3)?(e[3]||se.error(e[0]),e[4]=+(e[4]?e[5]+(e[6]||1):2*("even"===e[3]||"odd"===e[3])),e[5]=+(e[7]+e[8]||"odd"===e[3])):e[3]&&se.error(e[0]),e},PSEUDO:function(e){var t,n=!e[6]&&e[2];return J.CHILD.test(e[0])?null:(e[3]?e[2]=e[4]||e[5]||"":n&&X.test(n)&&(t=d(n,!0))&&(t=n.indexOf(")",n.length-t)-n.length)&&(e[0]=e[0].slice(0,t),e[2]=n.slice(0,t)),e.slice(0,3))}},filter:{TAG:function(e){var t=e.replace(re,f).toLowerCase();return"*"===e?function(){return!0}:function(e){return e.nodeName&&e.nodeName.toLowerCase()===t}},CLASS:function(e){var t=A[e+" "];return t||(t=new RegExp("(^|"+P+")"+e+"("+P+"|$)"))&&A(e,function(e){return t.test("string"==typeof e.className&&e.className||void 0!==e.getAttribute&&e.getAttribute("class")||"")})},ATTR:function(t,n,r){return function(e){e=se.attr(e,t);return null==e?"!="===n:!n||(e+="","="===n?e===r:"!="===n?e!==r:"^="===n?r&&0===e.indexOf(r):"*="===n?r&&-1:\x20\t\r\n\f]*)[\x20\t\r\n\f]*\/?>(?:<\/\1>|)$/i;function O(e,n,r){return b(n)?T.grep(e,function(e,t){return!!n.call(e,t,e)!==r}):n.nodeType?T.grep(e,function(e){return e===n!==r}):"string"!=typeof n?T.grep(e,function(e){return-1)[^>]*|#([\w-]+))$/;(T.fn.init=function(e,t,n){if(!e)return this;if(n=n||j,"string"!=typeof e)return e.nodeType?(this[0]=e,this.length=1,this):b(e)?void 0!==n.ready?n.ready(e):e(T):T.makeArray(e,this);if(!(r="<"===e[0]&&">"===e[e.length-1]&&3<=e.length?[null,e,null]:S.exec(e))||!r[1]&&t)return(!t||t.jquery?t||n:this.constructor(t)).find(e);if(r[1]){if(t=t instanceof T?t[0]:t,T.merge(this,T.parseHTML(r[1],t&&t.nodeType?t.ownerDocument||t:E,!0)),A.test(r[1])&&T.isPlainObject(t))for(var r in t)b(this[r])?this[r](t[r]):this.attr(r,t[r]);return this}return(e=E.getElementById(r[2]))&&(this[0]=e,this.length=1),this}).prototype=T.fn;var j=T(E),_=/^(?:parents|prev(?:Until|All))/,L={children:!0,contents:!0,next:!0,prev:!0};function I(e,t){for(;(e=e[t])&&1!==e.nodeType;);return e}T.fn.extend({has:function(e){var t=T(e,this),n=t.length;return this.filter(function(){for(var e=0;e\x20\t\r\n\f]*)/i,fe=/^$|^module$|\/(?:java|ecma)script/i;Vt=E.createDocumentFragment().appendChild(E.createElement("div")),(p=E.createElement("input")).setAttribute("type","radio"),p.setAttribute("checked","checked"),p.setAttribute("name","t"),Vt.appendChild(p),y.checkClone=Vt.cloneNode(!0).cloneNode(!0).lastChild.checked,Vt.innerHTML="",y.noCloneChecked=!!Vt.cloneNode(!0).lastChild.defaultValue,Vt.innerHTML="",y.option=!!Vt.lastChild;var pe={thead:[1,"","
    "],col:[2,"","
    "],tr:[2,"","
    "],td:[3,"","
    "],_default:[0,"",""]};function de(e,t){var n=void 0!==e.getElementsByTagName?e.getElementsByTagName(t||"*"):void 0!==e.querySelectorAll?e.querySelectorAll(t||"*"):[];return void 0===t||t&&N(e,t)?T.merge([e],n):n}function he(e,t){for(var n=0,r=e.length;n",""]);var ge=/<|&#?\w+;/;function ve(e,t,n,r,i){for(var o,a,s,u,l,c=t.createDocumentFragment(),f=[],p=0,d=e.length;p\s*$/g;function Ne(e,t){return N(e,"table")&&N(11!==t.nodeType?t:t.firstChild,"tr")&&T(e).children("tbody")[0]||e}function Ae(e){return e.type=(null!==e.getAttribute("type"))+"/"+e.type,e}function Oe(e){return"true/"===(e.type||"").slice(0,5)?e.type=e.type.slice(5):e.removeAttribute("type"),e}function Se(e,t){var n,r,i,o;if(1===t.nodeType){if(X.hasData(e)&&(o=X.get(e).events))for(i in X.remove(t,"handle events"),o)for(n=0,r=o[i].length;n").attr(n.scriptAttrs||{}).prop({charset:n.scriptCharset,src:n.url}).on("load error",i=function(e){r.remove(),i=null,e&&t("error"===e.type?404:200,e.type)}),E.head.appendChild(r[0])},abort:function(){i&&i()}}});var Vt,Xt=[],Yt=/(=)\?(?=&|$)|\?\?/;T.ajaxSetup({jsonp:"callback",jsonpCallback:function(){var e=Xt.pop()||T.expando+"_"+Nt.guid++;return this[e]=!0,e}}),T.ajaxPrefilter("json jsonp",function(e,t,n){var r,i,o,a=!1!==e.jsonp&&(Yt.test(e.url)?"url":"string"==typeof e.data&&0===(e.contentType||"").indexOf("application/x-www-form-urlencoded")&&Yt.test(e.data)&&"data");if(a||"jsonp"===e.dataTypes[0])return r=e.jsonpCallback=b(e.jsonpCallback)?e.jsonpCallback():e.jsonpCallback,a?e[a]=e[a].replace(Yt,"$1"+r):!1!==e.jsonp&&(e.url+=(At.test(e.url)?"&":"?")+e.jsonp+"="+r),e.converters["script json"]=function(){return o||T.error(r+" was not called"),o[0]},e.dataTypes[0]="json",i=C[r],C[r]=function(){o=arguments},n.always(function(){void 0===i?T(C).removeProp(r):C[r]=i,e[r]&&(e.jsonpCallback=t.jsonpCallback,Xt.push(r)),o&&b(i)&&i(o[0]),o=i=void 0}),"script"}),y.createHTMLDocument=((Vt=E.implementation.createHTMLDocument("").body).innerHTML="
    ",2===Vt.childNodes.length),T.parseHTML=function(e,t,n){return"string"!=typeof e?[]:("boolean"==typeof t&&(n=t,t=!1),t||(y.createHTMLDocument?((r=(t=E.implementation.createHTMLDocument("")).createElement("base")).href=E.location.href,t.head.appendChild(r)):t=E),r=!n&&[],(n=A.exec(e))?[t.createElement(n[1])]:(n=ve([e],t,r),r&&r.length&&T(r).remove(),T.merge([],n.childNodes)));var r},T.fn.load=function(e,t,n){var r,i,o,a=this,s=e.indexOf(" ");return-1").append(T.parseHTML(e)).find(r):e)}).always(n&&function(e,t){a.each(function(){n.apply(this,o||[e.responseText,t,e])})}),this},T.expr.pseudos.animated=function(t){return T.grep(T.timers,function(e){return t===e.elem}).length},T.offset={setOffset:function(e,t,n){var r,i,o,a,s=T.css(e,"position"),u=T(e),l={};"static"===s&&(e.style.position="relative"),o=u.offset(),r=T.css(e,"top"),a=T.css(e,"left"),a=("absolute"===s||"fixed"===s)&&-1<(r+a).indexOf("auto")?(i=(s=u.position()).top,s.left):(i=parseFloat(r)||0,parseFloat(a)||0),null!=(t=b(t)?t.call(e,n,T.extend({},o)):t).top&&(l.top=t.top-o.top+i),null!=t.left&&(l.left=t.left-o.left+a),"using"in t?t.using.call(e,l):u.css(l)}},T.fn.extend({offset:function(t){if(arguments.length)return void 0===t?this:this.each(function(e){T.offset.setOffset(this,t,e)});var e,n=this[0];return n?n.getClientRects().length?(e=n.getBoundingClientRect(),n=n.ownerDocument.defaultView,{top:e.top+n.pageYOffset,left:e.left+n.pageXOffset}):{top:0,left:0}:void 0},position:function(){if(this[0]){var e,t,n,r=this[0],i={top:0,left:0};if("fixed"===T.css(r,"position"))t=r.getBoundingClientRect();else{for(t=this.offset(),n=r.ownerDocument,e=r.offsetParent||n.documentElement;e&&(e===n.body||e===n.documentElement)&&"static"===T.css(e,"position");)e=e.parentNode;e&&e!==r&&1===e.nodeType&&((i=T(e).offset()).top+=T.css(e,"borderTopWidth",!0),i.left+=T.css(e,"borderLeftWidth",!0))}return{top:t.top-i.top-T.css(r,"marginTop",!0),left:t.left-i.left-T.css(r,"marginLeft",!0)}}},offsetParent:function(){return this.map(function(){for(var e=this.offsetParent;e&&"static"===T.css(e,"position");)e=e.offsetParent;return e||ne})}}),T.each({scrollLeft:"pageXOffset",scrollTop:"pageYOffset"},function(t,i){var o="pageYOffset"===i;T.fn[t]=function(e){return B(this,function(e,t,n){var r;return g(e)?r=e:9===e.nodeType&&(r=e.defaultView),void 0===n?r?r[i]:e[t]:void(r?r.scrollTo(o?r.pageXOffset:n,o?n:r.pageYOffset):e[t]=n)},t,e,arguments.length)}}),T.each(["top","left"],function(e,n){T.cssHooks[n]=Ve(y.pixelPosition,function(e,t){if(t)return t=Ue(e,n),Be.test(t)?T(e).position()[n]+"px":t})}),T.each({Height:"height",Width:"width"},function(a,s){T.each({padding:"inner"+a,content:s,"":"outer"+a},function(r,o){T.fn[o]=function(e,t){var n=arguments.length&&(r||"boolean"!=typeof e),i=r||(!0===e||!0===t?"margin":"border");return B(this,function(e,t,n){var r;return g(e)?0===o.indexOf("outer")?e["inner"+a]:e.document.documentElement["client"+a]:9===e.nodeType?(r=e.documentElement,Math.max(e.body["scroll"+a],r["scroll"+a],e.body["offset"+a],r["offset"+a],r["client"+a])):void 0===n?T.css(e,t,i):T.style(e,t,n,i)},s,n?e:void 0,n)}})}),T.each(["ajaxStart","ajaxStop","ajaxComplete","ajaxError","ajaxSuccess","ajaxSend"],function(e,t){T.fn[t]=function(e){return this.on(t,e)}}),T.fn.extend({bind:function(e,t,n){return this.on(e,null,t,n)},unbind:function(e,t){return this.off(e,null,t)},delegate:function(e,t,n,r){return this.on(t,e,n,r)},undelegate:function(e,t,n){return 1===arguments.length?this.off(e,"**"):this.off(t,e||"**",n)},hover:function(e,t){return this.mouseenter(e).mouseleave(t||e)}}),T.each("blur focus focusin focusout resize scroll click dblclick mousedown mouseup mousemove mouseover mouseout mouseenter mouseleave change select submit keydown keypress keyup contextmenu".split(" "),function(e,n){T.fn[n]=function(e,t){return 0":">",'"':""","'":"'","`":"`"},Ke=Fe(Be),$e=Fe(ye(Be)),We=te.templateSettings={evaluate:/<%([\s\S]+?)%>/g,interpolate:/<%=([\s\S]+?)%>/g,escape:/<%-([\s\S]+?)%>/g},ze=/(.)^/,Ue={"'":"'","\\":"\\","\r":"r","\n":"n","\u2028":"u2028","\u2029":"u2029"},Ve=/\\|'|\r|\n|\u2028|\u2029/g;function Xe(e){return"\\"+Ue[e]}var Ye=/^\s*(\w|\$)+\s*$/;var Je=0;function Ge(e,t,n,r,i){if(!(r instanceof t))return e.apply(n,i);n=Te(e.prototype),i=e.apply(n,i);return x(i)?i:n}var Qe=b(function(i,o){var a=Qe.placeholder,s=function(){for(var e=0,t=o.length,n=Array(t),r=0;r img { width: 30px; height: 30px; margin-right: 5px; margin-top: -5px; display: inline; } .bg-zone-local { background-color: rgb(221,255,221); } .bg-region-local { background-color: rgb(254,254,194); } .bg-global { background-color: white; } .popup { position: relative; display: inline-block; cursor: pointer; } /* The actual popup (appears on top) */ .popup .popuptext { visibility: hidden; width: 180px; height: 27px; background-color: white; color: #4b4b4b; text-align: center; border-radius: 6px; border-right: 1px solid #777; border-left: 1px solid #777; border-top: 1px solid #777; border-bottom: 1px solid #777; padding: 2px 0; position: absolute; z-index: 1; top: 60%; left: 50%; margin-left: -85%; } /* Popup arrow */ .popup .popuptext::after { content: ""; position: absolute; top: -5px; left: 50%; margin-left: -5px; width: 0; height: 0; border-left: 5px solid transparent; border-right: 5px solid transparent; border-bottom: 5px solid #777; } /* Toggle this class when clicking on the popup container (hide and show the popup) */ .popup .show { visibility: visible; -webkit-animation: fadeIn 1s; animation: fadeIn 1s } /* Add animation (fade in the popup) */ @-webkit-keyframes fadeIn { from {opacity: 0;} to {opacity: 1;} } @keyframes fadeIn { from {opacity: 0;} to {opacity:1 ;} } ================================================ FILE: nsqadmin/static/html/index.html ================================================ nsqadmin
    ================================================ FILE: nsqadmin/static/js/app_state.js ================================================ var Backbone = require('backbone'); var _ = require('underscore'); var AppState = Backbone.Model.extend({ defaults: function() { return { 'VERSION': VERSION, 'GRAPHITE_URL': GRAPHITE_URL, 'GRAPH_ENABLED': GRAPH_ENABLED, 'STATSD_INTERVAL': STATSD_INTERVAL, 'STATSD_COUNTER_FORMAT': STATSD_COUNTER_FORMAT, 'STATSD_GAUGE_FORMAT': STATSD_GAUGE_FORMAT, 'STATSD_PREFIX': STATSD_PREFIX, 'NSQLOOKUPD': NSQLOOKUPD, 'graph_interval': '2h', 'IS_ADMIN': IS_ADMIN, 'BASE_PATH': BASE_PATH }; }, initialize: function() { this.on('change:graph_interval', function(model, v) { localStorage.setItem('graph_interval', v); }); var qp = _.object(_.compact(_.map(window.location.search.slice(1).split('&'), function(item) { return item ? item.split('=') : false; }))); var def = this.get('GRAPH_ENABLED') ? '2h' : 'off'; var interval = qp['t'] || localStorage.getItem('graph_interval') || def; this.set('graph_interval', interval); }, basePath: function(p) { // if base path is / then don't prefix var bp = this.get('BASE_PATH') === '/' ? '' : this.get('BASE_PATH'); // remove trailing /, but guarantee at least / return (bp + p).replace(/\/$/, '') || '/'; }, apiPath: function(p) { return this.basePath('/api' + p); } }); var appState = new AppState(); window.AppState = appState; module.exports = appState; ================================================ FILE: nsqadmin/static/js/collections/nodes.js ================================================ var Backbone = require('backbone'); var AppState = require('../app_state'); var NodeModel = require('../models/node'); var Nodes = Backbone.Collection.extend({ model: NodeModel, comparator: 'id', constructor: function Nodes() { Backbone.Collection.prototype.constructor.apply(this, arguments); }, url: function() { return AppState.apiPath('/nodes'); }, parse: function(resp) { resp['nodes'].forEach(function(n) { var jaddr = n['broadcast_address']; if (jaddr.includes(':')) { // ipv6 raw address contains ':' // it must be wrapped in '[ ]' when joined with port jaddr = '[' + jaddr + ']'; } n['broadcast_address_http'] = jaddr + ':' + n['http_port']; }); return resp['nodes']; } }); module.exports = Nodes; ================================================ FILE: nsqadmin/static/js/collections/topics.js ================================================ var _ = require('underscore'); var Backbone = require('backbone'); var AppState = require('../app_state'); var Topic = require('../models/topic'); var Topics = Backbone.Collection.extend({ model: Topic, comparator: 'id', constructor: function Topics() { Backbone.Collection.prototype.constructor.apply(this, arguments); }, url: function() { return AppState.apiPath('/topics'); }, parse: function(resp) { var topics = _.map(resp['topics'], function(name) { return {'name': name}; }); return topics; } }); module.exports = Topics; ================================================ FILE: nsqadmin/static/js/lib/ajax_setup.js ================================================ var $ = require('jquery'); var _ = require('underscore'); // Set up some headers and options for every request. $.ajaxPrefilter(function(options) { options['headers'] = _.defaults(options['headers'] || {}, { 'X-UserAgent': USER_AGENT, 'Accept': 'application/vnd.nsq; version=1.0' }); options['timeout'] = 20 * 1000; options['contentType'] = 'application/json'; }); ================================================ FILE: nsqadmin/static/js/lib/handlebars_helpers.js ================================================ var $ = require('jquery'); var _ = require('underscore'); var Handlebars = require('hbsfy/runtime'); var AppState = require('../app_state'); var formatStatsdKey = function(metricType, key) { var fullKey = key; var fmt; if (metricType === 'counter') { fmt = AppState.get('STATSD_COUNTER_FORMAT'); fullKey = fmt.replace(/%s/g, key); } else if (metricType === 'gauge') { fmt = AppState.get('STATSD_GAUGE_FORMAT'); fullKey = fmt.replace(/%s/g, key); } return fullKey; }; var statsdPrefix = function(host) { var prefix = AppState.get('STATSD_PREFIX'); var statsdHostKey = host.replace(/[\.:]/g, '_'); prefix = prefix.replace(/%s/g, statsdHostKey); if (prefix.substring(prefix.length, 1) !== '.') { prefix += '.'; } return prefix; }; /* eslint-disable key-spacing */ var metricType = function(key) { return { 'depth': 'gauge', 'in_flight_count': 'gauge', 'deferred_count': 'gauge', 'requeue_count': 'counter', 'timeout_count': 'counter', 'message_count': 'counter', 'clients': 'gauge', '*_bytes': 'gauge', 'gc_pause_*': 'gauge', 'gc_runs': 'counter', 'heap_objects': 'gauge', 'e2e_processing_latency': 'gauge' }[key]; }; /* eslint-enable key-spacing */ var genColorList = function(typ, key) { if (typ === 'topic' || typ === 'channel') { if (key === 'depth' || key === 'deferred_count') { return 'red'; } } else if (typ === 'node') { return 'red,green,blue,purple'; } else if (typ === 'counter') { return 'green'; } return 'blue'; }; // sanitizeGraphiteKey removes special characters from a graphite key // this matches behavior of bitly/statsdaemon // eslint-disable-next-line max-len // https://github.com/bitly/statsdaemon/blob/fc46d9cfe29b674a0c8abc723afaa9370430cdcd/statsdaemon.go#L64-L88 var sanitizeGraphiteKey = function(s) { return s.replaceAll(' ', '_').replaceAll('/', '-').replaceAll(/[^a-zA-Z0-9-_.]/g, ''); }; var genTargets = function(typ, node, ns1, ns2, key) { var targets = []; var prefix = statsdPrefix(node ? node : '*'); var fullKey; var target; if (typ === 'topic') { fullKey = formatStatsdKey(metricType(key), prefix + 'topic.' + sanitizeGraphiteKey(ns1) + '.' + key); targets.push('sumSeries(' + fullKey + ')'); } else if (typ === 'channel') { fullKey = formatStatsdKey(metricType(key), prefix + 'topic.' + sanitizeGraphiteKey(ns1) + '.channel.' + sanitizeGraphiteKey(ns2) + '.' + key); targets.push('sumSeries(' + fullKey + ')'); } else if (typ === 'node') { target = prefix + 'mem.' + key; if (key === 'gc_runs') { target = 'movingAverage(' + target + ',45)'; } targets.push(formatStatsdKey(metricType(key), target)); } else if (typ === 'e2e') { targets = _.map(ns1['percentiles'], function(p) { var t; if (ns1['channel'] !== '') { t = prefix + 'topic.' + ns1['topic'] + '.channel.' + ns1['channel'] + '.' + key + '_' + (p['quantile'] * 100); } else { t = prefix + 'topic.' + ns1['topic'] + '.' + key + '_' + (p['quantile'] * 100); } if (node === '*') { t = 'averageSeries(' + t + ')'; } return 'scale(' + formatStatsdKey(metricType(key), t) + ',0.000001)'; }); } else if (typ === 'counter') { fullKey = formatStatsdKey(metricType(key), prefix + 'topic.*.channel.*.' + key); targets.push('sumSeries(' + fullKey + ')'); } return targets; }; Handlebars.registerHelper('default', function(x, defaultValue) { return x ? x : defaultValue; }); Handlebars.registerHelper('ifeq', function(a, b, options) { return (a === b) ? options.fn(this) : options.inverse(this); }); Handlebars.registerHelper('unlesseq', function(a, b, options) { return (a !== b) ? options.fn(this) : options.inverse(this); }); Handlebars.registerHelper('ifgteq', function(a, b, options) { return (a >= b) ? options.fn(this) : options.inverse(this); }); Handlebars.registerHelper('iflteq', function(a, b, options) { return (a <= b) ? options.fn(this) : options.inverse(this); }); Handlebars.registerHelper('length', function(xs) { return xs.length; }); Handlebars.registerHelper('lowercase', function(s) { return s.toLowerCase(); }); Handlebars.registerHelper('uppercase', function(s) { return s.toUpperCase(); }); // this helper is inclusive of the top number Handlebars.registerHelper('for', function(from, to, incr, block) { var accum = ''; for (var i = from; i <= to; i += incr) { accum += block.fn(i); } return accum; }); // Logical operators as helper functions, which can be useful when used within // an `if` or `unless` block via the new helper composition syntax, like so: // // {{#if (or step.unlocked step.is_finished)}} // Step is unlocked or finished! // {{/if}} // // Any number of arguments may be given to either helper. NOTE: _.initial() is // used below because every helper takes an options hash as its last argument. Handlebars.registerHelper('and', function() { return _.all(_.initial(arguments)); }); Handlebars.registerHelper('or', function() { return _.any(_.initial(arguments)); }); Handlebars.registerHelper('eq', function(a, b) { return a === b; }); Handlebars.registerHelper('neq', function(a, b) { return a !== b; }); Handlebars.registerHelper('urlencode', function(a) { return encodeURIComponent(a); }); Handlebars.registerHelper('floatToPercent', function(f) { return Math.floor(f * 100); }); Handlebars.registerHelper('floatToDecimalPercent', function(f) { return parseFloat((f * 100).toFixed(2)); }); Handlebars.registerHelper('percSuffix', function(f) { var v = Math.floor(f * 100) % 10; if (v === 1) { return 'st'; } else if (v === 2) { return 'nd'; } else if (v === 3) { return 'rd'; } return 'th'; }); Handlebars.registerHelper('commafy', function(n) { n = n || 0; return n.toString().replace(/\B(?=(\d{3})+(?!\d))/g, ','); }); function round(num, places) { var multiplier = Math.pow(10, places); return Math.round(num * multiplier) / multiplier; } Handlebars.registerHelper('nanotohuman', function(n) { var s = ''; var v; if (n >= 3600000000000) { v = Math.floor(n / 3600000000000); n = n % 3600000000000; s = v + 'h'; } if (n >= 60000000000) { v = Math.floor(n / 60000000000); n = n % 60000000000; s += v + 'm'; } if (n >= 1000000000) { n = round(n / 1000000000, 2); s += n + 's'; } else if (n >= 1000000) { n = round(n / 1000000, 2); s += n + 'ms'; } else if (n >= 1000) { n = round(n / 1000, 2); s += n + 'us'; } else { s = n + 'ns'; } return s; }); Handlebars.registerHelper('sparkline', function(typ, node, ns1, ns2, key) { var q = { 'colorList': genColorList(typ, key), 'height': '20', 'width': '120', 'hideGrid': 'true', 'hideLegend': 'true', 'hideAxes': 'true', 'bgcolor': 'ff000000', // transparent 'fgcolor': 'black', 'margin': '0', 'yMin': '0', 'lineMode': 'connected', 'drawNullAsZero': 'false', 'from': '-' + AppState.get('graph_interval'), 'until': '-1min' }; var interval = AppState.get('STATSD_INTERVAL') + 'sec'; q['target'] = _.map(genTargets(typ, node, ns1, ns2, key), function(t) { return 'summarize(' + t + ',"' + interval + '","avg")'; }); return AppState.get('GRAPHITE_URL') + '/render?' + $.param(q, true); }); Handlebars.registerHelper('large_graph', function(typ, node, ns1, ns2, key) { var q = { 'colorList': genColorList(typ, key), 'height': '450', 'width': '800', 'bgcolor': 'ff000000', // transparent 'fgcolor': '999999', 'yMin': '0', 'lineMode': 'connected', 'drawNullAsZero': 'false', 'from': '-' + AppState.get('graph_interval'), 'until': '-1min' }; var interval = AppState.get('STATSD_INTERVAL') + 'sec'; q['target'] = _.map(genTargets(typ, node, ns1, ns2, key), function(t) { if (metricType(key) === 'counter') { var scale = 1 / AppState.get('STATSD_INTERVAL'); t = 'scale(' + t + ',' + scale + ')'; } return 'summarize(' + t + ',"' + interval + '","avg")'; }); return AppState.get('GRAPHITE_URL') + '/render?' + $.param(q, true); }); Handlebars.registerHelper('rate', function(typ, node, ns1, ns2) { return genTargets(typ, node, ns1, ns2, 'message_count')[0]; }); Handlebars.registerPartial('error', require('../views/error.hbs')); Handlebars.registerPartial('warning', require('../views/warning.hbs')); Handlebars.registerHelper('basePath', function(p) { return AppState.basePath(p); }); ================================================ FILE: nsqadmin/static/js/lib/pubsub.js ================================================ var _ = require('underscore'); var Backbone = require('backbone'); var Pubsub = _.clone(Backbone.Events); // making this global to more easily trigger events from the console window.Pubsub = Pubsub; module.exports = Pubsub; ================================================ FILE: nsqadmin/static/js/main.js ================================================ var $ = require('jquery'); var Backbone = require('backbone'); // var Pubsub = require('./lib/pubsub'); var Router = require('./router'); var AppView = require('./views/app'); // When using browserify, we need to tell Backbone what jQuery to use. Backbone.$ = $; // Side effects: require('./lib/ajax_setup'); require('./lib/handlebars_helpers'); var start = function() { new AppView(); Router.start(); }; // Pubsub.on('all', function() { // console.log.apply(console, arguments); // }); start(); ================================================ FILE: nsqadmin/static/js/models/channel.js ================================================ var _ = require('underscore'); var AppState = require('../app_state'); var Backbone = require('backbone'); var Channel = Backbone.Model.extend({ idAttribute: 'name', constructor: function Channel() { Backbone.Model.prototype.constructor.apply(this, arguments); }, url: function() { return AppState.apiPath('/topics/' + encodeURIComponent(this.get('topic')) + '/' + encodeURIComponent(this.get('name'))); }, parse: function(response) { response['nodes'] = _.map(response['nodes'] || [], function(node) { var nodeParts = node['node'].split(':'); var port = nodeParts.pop(); var address = nodeParts.join(':'); var hostname = node['hostname']; var zonecount = node['zone_local_msg_count']; var deliverycount = node['delivery_msg_count']; var regioncount = node['region_local_msg_count']; var globalcount = node['global_msg_count']; node['show_broadcast_address'] = hostname.toLowerCase() !== address.toLowerCase(); node['hostname_port'] = hostname + ':' + port; node['zone_local_percentage'] = zonecount / deliverycount; node['region_local_percentage'] = regioncount / deliverycount; node['global_percentage'] = globalcount / deliverycount; if (isNaN(node['zone_local_percentage'])) { node['zone_local_percentage'] = 0; } if (isNaN(node['region_local_percentage'])) { node['region_local_percentage'] = 0; } if (isNaN(node['global_percentage'])) { node['global_percentage'] = 0; } return node; }); response['clients'] = _.map(response['clients'] || [], function(client) { var clientId = client['client_id']; var hostname = client['hostname']; var shortHostname = hostname.split('.')[0]; // ignore client_id if it's duplicative client['show_client_id'] = (clientId.toLowerCase() !== shortHostname.toLowerCase() && clientId.toLowerCase() !== hostname.toLowerCase()); var port = client['remote_address'].split(':').pop(); client['hostname_port'] = hostname + ':' + port; return client; }); return response; } }); module.exports = Channel; ================================================ FILE: nsqadmin/static/js/models/node.js ================================================ var AppState = require('../app_state'); var Backbone = require('backbone'); var NodeModel = Backbone.Model.extend({ idAttribute: 'name', constructor: function Node() { Backbone.Model.prototype.constructor.apply(this, arguments); }, urlRoot: function() { return AppState.apiPath('/nodes'); }, tombstoneTopic: function(topic) { return this.destroy({ 'data': JSON.stringify({'topic': topic}), 'dataType': 'text' }); } }); module.exports = NodeModel; ================================================ FILE: nsqadmin/static/js/models/topic.js ================================================ var _ = require('underscore'); var AppState = require('../app_state'); var Backbone = require('backbone'); var Topic = Backbone.Model.extend({ idAttribute: 'name', constructor: function Topic() { Backbone.Model.prototype.constructor.apply(this, arguments); }, url: function() { return AppState.apiPath('/topics/' + encodeURIComponent(this.get('name'))); }, parse: function(response) { response['nodes'] = _.map(response['nodes'] || [], function(node) { var nodeParts = node['node'].split(':'); var port = nodeParts.pop(); var address = nodeParts.join(':'); var hostname = node['hostname']; node['show_broadcast_address'] = hostname.toLowerCase() !== address.toLowerCase(); node['hostname_port'] = hostname + ':' + port; return node; }); return response; } }); module.exports = Topic; ================================================ FILE: nsqadmin/static/js/router.js ================================================ var Backbone = require('backbone'); var AppState = require('./app_state'); var Pubsub = require('./lib/pubsub'); var Router = Backbone.Router.extend({ initialize: function() { var bp = function(p) { // remove leading slash return AppState.basePath(p).substring(1); }; this.route(bp('/'), 'topics'); this.route(bp('/topics/(:topic)(/:channel)'), 'topic'); this.route(bp('/lookup'), 'lookup'); this.route(bp('/nodes(/:node)'), 'nodes'); this.route(bp('/counter'), 'counter'); // this.listenTo(this, 'route', function(route, params) { // console.log('Route: %o; params: %o', route, params); // }); }, start: function() { Backbone.history.start({ 'pushState': true }); }, topics: function() { Pubsub.trigger('topics:show'); }, topic: function(topic, channel) { if (channel !== null) { Pubsub.trigger('channel:show', topic, channel); return; } Pubsub.trigger('topic:show', topic); }, lookup: function() { Pubsub.trigger('lookup:show'); }, nodes: function(node) { if (node !== null) { Pubsub.trigger('node:show', node); return; } Pubsub.trigger('nodes:show'); }, counter: function() { Pubsub.trigger('counter:show'); } }); module.exports = new Router(); ================================================ FILE: nsqadmin/static/js/views/app.js ================================================ var $ = require('jquery'); window.jQuery = $; var bootstrap = require('bootstrap'); //eslint-disable-line no-unused-vars var bootbox = require('bootbox'); var AppState = require('../app_state'); var Pubsub = require('../lib/pubsub'); var Router = require('../router'); var BaseView = require('./base'); var HeaderView = require('./header'); var TopicsView = require('./topics'); var TopicView = require('./topic'); var ChannelView = require('./channel'); var LookupView = require('./lookup'); var NodesView = require('./nodes'); var NodeView = require('./node'); var CounterView = require('./counter'); var NodeModel = require('../models/node'); var TopicModel = require('../models/topic'); var ChannelModel = require('../models/channel'); var AppView = BaseView.extend({ // not a fan of setting a view's el to an existing element on the page // for the top-level AppView, it seems appropriate, however el: '#container', events: { 'click .link': 'onLinkClick', 'click .tombstone-link': 'onTombstoneClick' }, initialize: function() { BaseView.prototype.initialize.apply(this, arguments); this.listenTo(Pubsub, 'topics:show', this.showTopics); this.listenTo(Pubsub, 'topic:show', this.showTopic); this.listenTo(Pubsub, 'channel:show', this.showChannel); this.listenTo(Pubsub, 'lookup:show', this.showLookup); this.listenTo(Pubsub, 'nodes:show', this.showNodes); this.listenTo(Pubsub, 'node:show', this.showNode); this.listenTo(Pubsub, 'counter:show', this.showCounter); this.listenTo(Pubsub, 'view:ready', function() { $('.rate').each(function(i, el) { var $el = $(el); var interval = AppState.get('STATSD_INTERVAL'); var q = { 'target': $el.attr('target'), 'from': '-' + (2 * interval) + 'sec', 'until': '-' + interval + 'sec', 'format': 'json', }; var formatRate = function(data) { if (data[0] === null || data[0]['datapoints'][0] === null || data[0]['datapoints'][0][0] < 0) { return 'N/A'; } else { return (data[0]['datapoints'][0][0] / interval).toFixed(2); } }; $.ajax({ url: AppState.get('GRAPHITE_URL') + '/render', data: q, dataType: 'jsonp', jsonp: 'jsonp' }) .done(function(data) { $el.html(formatRate(data)); }) .fail(function() { $el.html('ERROR'); }); }); }); this.render(); }, postRender: function() { this.appendSubview(new HeaderView()); }, showView: function(f) { window.scrollTo(0, 0); if (this.currentView) { this.currentView.remove(); } this.currentView = f(); this.appendSubview(this.currentView); }, showTopics: function() { this.showView(function() { return new TopicsView(); }); }, showTopic: function(topic) { this.showView(function() { var model = new TopicModel({'name': topic, 'isAdmin': AppState.get('IS_ADMIN')}); return new TopicView({'model': model}); }); }, showChannel: function(topic, channel) { this.showView(function() { var model = new ChannelModel({ 'topic': topic, 'name': channel, 'isAdmin': AppState.get('IS_ADMIN') }); return new ChannelView({'model': model}); }); }, showLookup: function() { this.showView(function() { return new LookupView({'isAdmin': AppState.get('IS_ADMIN')}); }); }, showNodes: function() { this.showView(function() { return new NodesView(); }); }, showNode: function(node) { this.showView(function() { var model = new NodeModel({'name': node}); return new NodeView({'model': model}); }); }, showCounter: function() { this.showView(function() { return new CounterView(); }); }, onLinkClick: function(e) { if (e.ctrlKey || e.metaKey) { // allow ctrl+click to open in a new tab return; } e.preventDefault(); e.stopPropagation(); Router.navigate($(e.currentTarget).attr('href'), {'trigger': true}); }, onTombstoneClick: function(e) { e.preventDefault(); e.stopPropagation(); var nodeName = $(e.target).data('node'); var topicName = $(e.target).data('topic'); var txt = 'Are you sure you want to tombstone ' + nodeName + '?'; bootbox.confirm(txt, function(result) { if (result !== true) { return; } var node = new NodeModel({ 'name': nodeName }); node.tombstoneTopic(topicName) .done(function() { window.location.reload(true); }) .fail(this.handleAJAXError.bind(this)); }.bind(this)); } }); module.exports = AppView; ================================================ FILE: nsqadmin/static/js/views/base.js ================================================ var $ = require('jquery'); var _ = require('underscore'); var Backbone = require('backbone'); var AppState = require('../app_state'); var errorTemplate = require('./error.hbs'); var BaseView = Backbone.View.extend({ constructor: function(options) { // As of 1.10, Backbone no longer automatically attaches options passed // to the constructor as this.options, but that's often useful in some // cases, like a className function, that happen before initialize() // would have a chance to attach the same options. this.options = options || {}; return Backbone.View.prototype.constructor.apply(this, arguments); }, initialize: function() { this.subviews = []; this.rendered = false; }, template: function() {}, skippedRender: function() {}, render: function(data) { if (this.renderOnce && this.rendered) { this.skippedRender(); return this; } this.removeSubviews(); var ctx = this.getRenderCtx(data); // console.log('render ctx: %o', ctx); var html = this.template(ctx); if (!this.removed) { this.$el.empty(); this.$el.append(html); this.postRender(ctx); } this.rendered = true; return this; }, getRenderCtx: function(data) { var ctx = { 'graph_enabled': AppState.get('GRAPH_ENABLED'), 'graph_interval': AppState.get('graph_interval'), 'graph_active': AppState.get('GRAPH_ENABLED') && AppState.get('graph_interval') !== 'off', 'nsqlookupd': AppState.get('NSQLOOKUPD'), 'version': AppState.get('VERSION') }; if (this.model) { ctx = _.extend(ctx, this.model.toJSON()); } else if (this.collection) { ctx = _.extend(ctx, {'collection': this.collection.toJSON()}); } if (data) { ctx = _.extend(ctx, data); } return ctx; }, postRender: function() {}, appendSubview: function(subview, selector) { return this.appendSubviews([subview], selector); }, appendSubviews: function(subviews, selector) { this.subviews.push.apply(this.subviews, subviews); var $el = selector ? this.$(selector) : this.$el; $el.append(subviews.map(function(subview) { return subview.render().delegateEvents().el; })); }, removeSubviews: function() { while (this.subviews.length) { this.subviews.pop().remove(); } }, remove: function() { this.removed = true; this.removeSubviews(); Backbone.View.prototype.remove.apply(this, arguments); }, parseErrorMessage: function(jqXHR) { var msg = 'ERROR: failed to connect to nsqadmin'; if (jqXHR.readyState === 4) { try { var parsed = JSON.parse(jqXHR.responseText); msg = parsed['message']; } catch (err) { msg = 'ERROR: failed to decode JSON - ' + err.message; } } return msg; }, handleAJAXError: function(jqXHR) { $('#warning, #error').hide(); $('#error .alert').text(this.parseErrorMessage(jqXHR)); $('#error').show(); }, handleViewError: function(jqXHR) { this.removeSubviews(); this.$el.html(errorTemplate({'message': this.parseErrorMessage(jqXHR)})); } }); module.exports = BaseView; ================================================ FILE: nsqadmin/static/js/views/channel.hbs ================================================ {{> warning}} {{> error}}
    1. Streams
    2. {{topic}}
    3. {{name}}

    Topic: {{topic}}

    Channel: {{name}}

    {{#unless nodes.length}}

    Notice

    No producers exist for this topic/channel.

    See Lookup for more information.

    {{else}} {{#if isAdmin}}
    {{#if paused}} {{else}} {{/if}}
    {{/if}}

    Channel

    {{#if e2e_processing_latency.percentiles.length}} {{/if}} {{#if graph_active}}{{/if}} {{#if delivery_msg_count}}{{/if}} {{#each e2e_processing_latency.percentiles}} {{/each}} {{#each nodes}} {{#if ../graph_active}} {{/if}} {{#if delivery_msg_count}} {{/if}} {{#if e2e_processing_latency.percentiles.length}} {{#each e2e_processing_latency.percentiles}} {{/each}} {{/if}} {{#if ../graph_active}} {{#if delivery_msg_count}}{{/if}} {{#if e2e_processing_latency.percentiles.length}} {{/if}} {{/if}} {{/each}} {{#if graph_active}} {{/if}} {{#if delivery_msg_count}}{{/if}} {{#if e2e_processing_latency.percentiles.length}} {{#each e2e_processing_latency.percentiles}} {{/each}} {{/if}} {{#if graph_active}} {{#if delivery_msg_count}}{{/if}} {{#if e2e_processing_latency.percentiles.length}} {{/if}} {{/if}}
      Message Queues StatisticsE2E Processing Latency
    NSQd Host Depth Memory + Disk In-Flight Deferred Requeued Timed Out MessagesRateConnectionsDelivery{{floatToPercent quantile}}{{percSuffix quantile}}
    {{#if show_broadcast_address}} {{hostname_port}} ({{node}}) {{else}} {{hostname_port}} {{/if}} {{#if paused}} paused{{/if}} {{commafy depth}} {{commafy memory_depth}} + {{commafy backend_depth}} {{commafy in_flight_count}} {{commafy deferred_count}} {{commafy requeue_count}} {{commafy timeout_count}} {{commafy message_count}}{{commafy client_count}}
    {{floatToPercent zone_local_percentage}}%
    {{floatToPercent region_local_percentage}}%
    {{floatToPercent global_percentage}}%
    {{floatToDecimalPercent zone_local_percentage}}% | {{floatToDecimalPercent region_local_percentage}}% | {{floatToDecimalPercent global_percentage}}%
    		{{nanotohuman average}}
    Total: {{commafy depth}} {{commafy memory_depth}} + {{commafy backend_depth}} {{commafy in_flight_count}} {{commafy deferred_count}} {{commafy requeue_count}} {{commafy timeout_count}} {{commafy message_count}}{{commafy client_count}} {{nanotohuman average}}
    {{/unless}}

    Client Connections

    {{#unless clients.length}}

    Notice

    No clients connected to this channel
    {{else}} {{#each clients}} {{/each}}
    Client Host User-Agent Attributes NSQd Host In-Flight Ready Count Finished Requeued Messages Connected
    {{hostname_port}}{{#if show_client_id}} ({{client_id}}){{/if}} {{#if user_agent.length}}{{user_agent}}{{/if}} {{#if sample_rate}} Sampled {{sample_rate}}% {{/if}} {{#if tls}} TLS {{/if}} {{#if deflate}} Deflate {{/if}} {{#if snappy}} Snappy {{/if}} {{#if authed}} {{#if auth_identity_url}}{{/if}} {{#if auth_identity_url}}{{/if}} {{/if}} {{#if topology_region}} {{topology_region}} {{/if}} {{#if topology_zone}} {{topology_zone}} {{/if}} {{#if (and node_topology_zone (eq topology_zone node_topology_zone))}} zoneLocal {{else}} {{#if (and node_topology_region (eq topology_region node_topology_region))}} regionLocal {{/if}} {{/if}} {{node}} {{commafy in_flight_count}} {{commafy ready_count}} {{commafy finish_count}} {{commafy requeue_count}} {{commafy message_count}} {{nanotohuman connected}}
    {{/unless}}
    ================================================ FILE: nsqadmin/static/js/views/channel.js ================================================ var $ = require('jquery'); window.jQuery = $; var bootstrap = require('bootstrap'); //eslint-disable-line no-unused-vars var bootbox = require('bootbox'); var Pubsub = require('../lib/pubsub'); var AppState = require('../app_state'); var BaseView = require('./base'); var ChannelView = BaseView.extend({ className: 'channel container-fluid', template: require('./spinner.hbs'), events: { 'click .channel-actions button': 'channelAction', 'click .popup': 'showDeliveryBreakdown' }, initialize: function() { BaseView.prototype.initialize.apply(this, arguments); this.listenTo(AppState, 'change:graph_interval', this.render); var isAdmin = this.model.get('isAdmin'); this.model.fetch() .done(function(data) { this.template = require('./channel.hbs'); this.render({'message': data['message'], 'isAdmin': isAdmin}); }.bind(this)) .fail(this.handleViewError.bind(this)) .always(Pubsub.trigger.bind(Pubsub, 'view:ready')); }, showDeliveryBreakdown: function(e) { e.preventDefault(); e.stopPropagation(); var popup = document.getElementById($(e.currentTarget).data('id')); popup.classList.toggle("show"); }, channelAction: function(e) { e.preventDefault(); e.stopPropagation(); var action = $(e.currentTarget).data('action'); var txt = 'Are you sure you want to ' + action + ' ' + this.model.get('topic') + '/' + this.model.get('name') + '?'; bootbox.confirm(txt, function(result) { if (result !== true) { return; } if (action === 'delete') { var topic = this.model.get('topic'); $.ajax(this.model.url(), {'method': 'DELETE'}) .done(function() { window.location = AppState.basePath('/topics/' + encodeURIComponent(topic)); }) .fail(this.handleAJAXError.bind(this)); } else { $.post(this.model.url(), JSON.stringify({'action': action})) .done(function() { window.location.reload(true); }) .fail(this.handleAJAXError.bind(this)); } }.bind(this)); } }); module.exports = ChannelView; ================================================ FILE: nsqadmin/static/js/views/counter.hbs ================================================ {{> warning}} {{> error}}

    Messages Processed

    {{#if graph_active}} {{/if}}
    ================================================ FILE: nsqadmin/static/js/views/counter.js ================================================ var _ = require('underscore'); var $ = require('jquery'); var AppState = require('../app_state'); var BaseView = require('./base'); var CounterView = BaseView.extend({ className: 'counter container-fluid', template: require('./counter.hbs'), initialize: function() { BaseView.prototype.initialize.apply(this, arguments); this.listenTo(AppState, 'change:graph_interval', function() { clearTimeout(this.poller); clearTimeout(this.animator); this.render(); this.start(); }); this.start(); }, remove: function() { clearTimeout(this.poller); clearTimeout(this.animator); BaseView.prototype.remove.apply(this, arguments); }, start: function() { this.poller = null; this.animator = null; this.delta = 0; this.looping = false; this.targetPollInterval = 10000; this.currentNum = -1; this.lastNum = 0; this.interval = 100; this.graphUrl = null; this.updateStats(); }, startLoop: function(i) { this.interval = i; this.poller = setTimeout(this.updateStats.bind(this), i); }, updateStats: function() { $.get(AppState.apiPath('/counter')).done(function(data) { if (this.removed) { return; } var num = _.reduce(data['stats'], function(n, v) { return n + v['message_count']; }, 0); if (this.currentNum === -1) { // seed the display this.currentNum = num; this.lastNum = num; this.writeCounts(this.currentNum); } else if (num > this.lastNum) { var delta = num - this.lastNum; this.delta = (delta / (this.interval / 1000)) / 50; this.lastNum = num; if (!this.animator) { this.displayFrame(); } } var newInterval = this.interval; if (newInterval < this.targetPollInterval) { newInterval = this.interval + 1000; } this.startLoop(newInterval); $('#warning, #error').hide(); if (data['message'] !== '') { $('#warning .alert').text(data['message']); $('#warning').show(); } }.bind(this)).fail(function(jqXHR) { if (this.removed) { return; } clearTimeout(this.animator); this.animator = null; this.startLoop(10000); this.handleAJAXError(jqXHR); }.bind(this)); if ($('#big_graph').length) { if (!this.graphUrl) { this.graphUrl = $('#big_graph').attr('src'); } var uniq = Math.floor(Math.random() * 1000000); $('#big_graph').attr('src', this.graphUrl + '&_uniq=' + uniq); } }, displayFrame: function() { this.currentNum = Math.min(this.currentNum + this.delta, this.lastNum); this.writeCounts(this.currentNum); if (this.currentNum < this.lastNum) { this.animator = setTimeout(this.displayFrame.bind(this), 1000 / 60); } else { this.animator = null; } }, writeCounts: function(c) { var text = parseInt(c, 10).toString(); var node = $('.numbers')[0]; var n = $('.numbers .number'); for (var i = 0; i < text.length; i++) { var v = text.charAt(i); if (n.length > i) { var el = $(n[i]); el.show(); el.find('.top').text(v); el.find('.bottom').text(v); } else { $(node).append('' + v + '' + v + '\n'); } } $('.numbers .number').each(function(ii, vv) { if (ii >= text.length) { $(vv).hide(); } }); } }); module.exports = CounterView; ================================================ FILE: nsqadmin/static/js/views/error.hbs ================================================
    {{message}}
    ================================================ FILE: nsqadmin/static/js/views/header.hbs ================================================ ================================================ FILE: nsqadmin/static/js/views/header.js ================================================ var _ = require('underscore'); var $ = require('jquery'); var AppState = require('../app_state'); var BaseView = require('./base'); var HeaderView = BaseView.extend({ className: 'header', template: require('./header.hbs'), events: { 'click .dropdown-menu li': 'onGraphIntervalClick' }, initialize: function() { BaseView.prototype.initialize.apply(this, arguments); this.listenTo(AppState, 'change:graph_interval', this.render); }, getRenderCtx: function() { return _.extend(BaseView.prototype.getRenderCtx.apply(this, arguments), { 'graph_intervals': ['1h', '2h', '12h', '24h', '48h', '168h', 'off'], 'graph_interval': AppState.get('graph_interval') }); }, onReset: function() { this.render(); this.$('.dropdown-toggle').dropdown(); }, onGraphIntervalClick: function(e) { e.stopPropagation(); AppState.set('graph_interval', $(e.target).text()); } }); module.exports = HeaderView; ================================================ FILE: nsqadmin/static/js/views/lookup.hbs ================================================ {{> warning}} {{> error}}

    Lookup

    {{#unless nsqlookupd.length}}

    Notice

    nsqadmin is not configured with nsqlookupd hosts
    {{else}}
    {{#each nsqlookupd}} {{/each}}
    nsqlookupd Host
    {{this}}
    {{#if topics}}
    Below is a tree of Topics/Channels that are currently inactive (i.e. not produced on any nsqd in the cluster but are present in the lookup data)
      {{#each topics}}
    • {{name}}
        {{#each channels}}
      • {{this}}
        • {{/each}}
      • {{/each}}
    {{else}}

    Notice

    No inactive Topics
    {{/if}}
    {{#if isAdmin}}
    Create Topic/Channel

    This provides a way to setup a stream hierarchy before services are deployed to production.

    If Channel Name is empty, just the topic is created.

    {{/if}} {{/unless}} ================================================ FILE: nsqadmin/static/js/views/lookup.js ================================================ var _ = require('underscore'); var $ = require('jquery'); var AppState = require('../app_state'); var Pubsub = require('../lib/pubsub'); var BaseView = require('./base'); var Topic = require('../models/topic'); var Channel = require('../models/channel'); var LookupView = BaseView.extend({ className: 'lookup container-fluid', template: require('./spinner.hbs'), events: { 'click .hierarchy button': 'onCreateTopicChannel', 'click .delete-topic-link': 'onDeleteTopic', 'click .delete-channel-link': 'onDeleteChannel' }, initialize: function() { BaseView.prototype.initialize.apply(this, arguments); var isAdmin = arguments[0]['isAdmin']; $.ajax(AppState.apiPath('/topics?inactive=true')) .done(function(data) { this.template = require('./lookup.hbs'); this.render({ 'topics': _.map(data['topics'], function(v, k) { return {'name': k, 'channels': v}; }), 'message': data['message'], 'isAdmin': isAdmin }); }.bind(this)) .fail(this.handleViewError.bind(this)) .always(Pubsub.trigger.bind(Pubsub, 'view:ready')); }, onCreateTopicChannel: function(e) { e.preventDefault(); e.stopPropagation(); var topic = $(e.target.form.elements['topic']).val(); var channel = $(e.target.form.elements['channel']).val(); if (topic === '' && channel === '') { return; } $.post(AppState.apiPath('/topics'), JSON.stringify({ 'topic': topic, 'channel': channel })) .done(function() { window.location.reload(true); }) .fail(this.handleAJAXError.bind(this)); }, onDeleteTopic: function(e) { e.preventDefault(); e.stopPropagation(); var topic = new Topic({ 'name': $(e.target).data('topic') }); topic.destroy({'dataType': 'text'}) .done(function() { window.location.reload(true); }) .fail(this.handleAJAXError.bind(this)); }, onDeleteChannel: function(e) { e.preventDefault(); e.stopPropagation(); var channel = new Channel({ 'topic': $(e.target).data('topic'), 'name': $(e.target).data('channel') }); channel.destroy({'dataType': 'text'}) .done(function() { window.location.reload(true); }) .fail(this.handleAJAXError.bind(this)); } }); module.exports = LookupView; ================================================ FILE: nsqadmin/static/js/views/node.hbs ================================================ {{> warning}} {{> error}}
    1. Nodes
    2. {{name}}
    {{#if graph_active}}
    GC Pressure
    GC Pause Percentiles
    GC Runs
    Heap Objects In-Use
    {{/if}}
    {{#unless topics.length}}

    Notice

    No topics exist on this node.
    {{else}} {{#each topics}} {{#unless channels.length}} {{else}} {{#each channels}} {{#unless clients.length}} {{else}} {{#each clients}} {{/each}} {{/unless}} {{/each}} {{/unless}} {{/each}}
    {{../topics.length}} Topics {{commafy total_messages}} Messages {{commafy total_clients}} Clients
    Topic Depth Memory + Disk Messages Channels
    {{topic_name}} {{#if paused}}paused{{/if}} {{#if ../graph_active}}{{/if}} {{commafy depth}} {{commafy memory_depth}} + {{commafy backend_depth}} {{#if ../graph_active}}{{/if}} {{commafy message_count}} {{commafy channels.length}}

    Notice

    No channels exist for this topic.
    Channel Depth Memory + Disk In-Flight Deferred Requeued Timed Out Messages Connections
    {{channel_name}} {{#if paused}}paused{{/if}} {{#if ../../../graph_active}}{{/if}} {{commafy depth}} {{commafy memory_depth}} + {{commafy backend_depth}} {{commafy in_flight_count}} {{commafy deferred_count}} {{commafy requeue_count}} {{commafy timeout_count}} {{commafy message_count}} {{#if ../../../graph_active}}{{/if}} {{commafy clients.length}}

    Notice

    No clients connected to this channel.
    Client Host User-Agent Attributes In-Flight Ready Count Requeued Finished Messages Connected
    {{hostname}}{{#if show_client_id}} ({{client_id}}){{/if}} {{#if user_agent.length}}{{user_agent}}{{/if}} {{#if sample_rate}} Sampled {{sample_rate}}% {{/if}} {{#if tls}} TLS {{/if}} {{#if deflate}} Deflate {{/if}} {{#if snappy}} Snappy {{/if}} {{#if authed}} {{#if auth_identity_url}}{{/if}} {{#if auth_identity_url}}{{/if}} {{/if}} {{commafy in_flight_count}} {{commafy ready_count}} {{commafy requeue_count}} {{commafy finish_count}} {{commafy message_count}} {{nanotohuman connected}}
    {{/unless}}
    ================================================ FILE: nsqadmin/static/js/views/node.js ================================================ var Pubsub = require('../lib/pubsub'); var AppState = require('../app_state'); var BaseView = require('./base'); var NodeView = BaseView.extend({ className: 'node container-fluid', template: require('./spinner.hbs'), initialize: function() { BaseView.prototype.initialize.apply(this, arguments); this.listenTo(AppState, 'change:graph_interval', this.render); this.model.fetch() .done(function(data) { this.template = require('./node.hbs'); this.render({'message': data['message']}); }.bind(this)) .fail(this.handleViewError.bind(this)) .always(Pubsub.trigger.bind(Pubsub, 'view:ready')); } }); module.exports = NodeView; ================================================ FILE: nsqadmin/static/js/views/nodes.hbs ================================================ {{> warning}} {{> error}}

    NSQd Nodes ({{collection.length}})

    {{#if nsqlookupd.length}} {{/if}} {{#each collection}} {{#if ../nsqlookupd.length}} {{/if}} {{/each}}
    Hostname Broadcast Address TCP Port HTTP Port Version Region ZoneLookupd Conns.Topics
    {{hostname}} {{broadcast_address}} {{tcp_port}} {{http_port}} {{version}} {{topology_region}} {{topology_zone}} {{remote_addresses.length}}
    {{#each remote_addresses}}{{this}}
    {{/each}}
    		{{#if topics.length}} {{topics.length}} {{#each topics}} {{topic}} {{/each}} {{/if}}
    ================================================ FILE: nsqadmin/static/js/views/nodes.js ================================================ var $ = require('jquery'); var Pubsub = require('../lib/pubsub'); var AppState = require('../app_state'); var BaseView = require('./base'); var Nodes = require('../collections/nodes'); var NodesView = BaseView.extend({ className: 'nodes container-fluid', template: require('./spinner.hbs'), events: { 'click .conn-count': 'onClickConnCount' }, initialize: function() { BaseView.prototype.initialize.apply(this, arguments); this.listenTo(AppState, 'change:graph_interval', this.render); this.collection = new Nodes(); this.collection.fetch() .done(function(data) { this.template = require('./nodes.hbs'); this.render({'message': data['message']}); }.bind(this)) .fail(this.handleViewError.bind(this)) .always(Pubsub.trigger.bind(Pubsub, 'view:ready')); }, onClickConnCount: function(e) { e.preventDefault(); $(e.target).next().toggle(); } }); module.exports = NodesView; ================================================ FILE: nsqadmin/static/js/views/spinner.hbs ================================================
    ================================================ FILE: nsqadmin/static/js/views/topic.hbs ================================================ {{> warning}} {{> error}}
    1. Streams
    2. {{name}}

    Topic: {{name}}

    {{#unless nodes.length}}

    Topic Message Queue

    Notice

    No producers exist for this topic.

    See Lookup for more information.

    {{else}} {{#if isAdmin}}
    {{#if paused}} {{else}} {{/if}}
    {{/if}}

    Topic Message Queue

    {{#if e2e_processing_latency.percentiles.length}} {{/if}} {{#if graph_active}}{{/if}} {{#each e2e_processing_latency.percentiles}} {{/each}} {{#each nodes}} {{#if ../graph_active}} {{/if}} {{#if e2e_processing_latency.percentiles.length}} {{#each e2e_processing_latency.percentiles}} {{/each}} {{/if}} {{#if ../graph_active}} {{#if e2e_processing_latency.percentiles.length}} {{/if}} {{/if}} {{/each}} {{#if graph_active}}{{/if}} {{#if e2e_processing_latency.percentiles.length}} {{#each e2e_processing_latency.percentiles}} {{/each}} {{/if}} {{#if graph_active}} {{#if e2e_processing_latency.percentiles.length}} {{/if}} {{/if}}
    E2E Processing Latency
    NSQd Host Depth Memory + Disk MessagesRateChannels{{floatToPercent quantile}}{{percSuffix quantile}}
    {{#if show_broadcast_address}} {{hostname_port}} ({{node}}) {{else}} {{hostname_port}} {{/if}} {{#if paused}} paused{{/if}} {{commafy depth}} {{commafy memory_depth}} + {{commafy backend_depth}} {{commafy message_count}}{{commafy this/channels.length}} {{nanotohuman average}}
    Total: {{commafy depth}} {{commafy memory_depth}} + {{commafy backend_depth}} {{commafy message_count}}{{commafy channels.length}} {{nanotohuman average}}
    {{/unless}}
    {{#unless channels.length}}

    Channel Message Queues

    Notice

    No channels exist for this topic.

    Messages will queue at the topic until a channel is created.

    {{else}}

    Channel Message Queues

    {{#if e2e_processing_latency.percentiles.length}} {{/if}} {{#each e2e_processing_latency.percentiles}} {{/each}} {{#each channels}} {{#if e2e_processing_latency.percentiles.length}} {{#each e2e_processing_latency.percentiles}} {{/each}} {{/if}} {{#if ../graph_active}} {{#if e2e_processing_latency.percentiles.length}} {{/if}} {{/if}} {{/each}}
    E2E Processing Latency
    Channel Depth Memory + Disk In-Flight Deferred Requeued Timed Out Messages Connections{{floatToPercent quantile}}{{percSuffix quantile}}
    {{channel_name}} {{#if paused}}paused{{/if}} {{commafy depth}} {{commafy memory_depth}} + {{commafy backend_depth}} {{commafy in_flight_count}} {{commafy deferred_count}} {{commafy requeue_count}} {{commafy timeout_count}} {{commafy message_count}} {{commafy client_count}} {{nanotohuman average}}
    {{/unless}}
    ================================================ FILE: nsqadmin/static/js/views/topic.js ================================================ var $ = require('jquery'); window.jQuery = $; var bootstrap = require('bootstrap'); //eslint-disable-line no-unused-vars var bootbox = require('bootbox'); var Pubsub = require('../lib/pubsub'); var AppState = require('../app_state'); var BaseView = require('./base'); var TopicView = BaseView.extend({ className: 'topic container-fluid', template: require('./spinner.hbs'), events: { 'click .topic-actions button': 'topicAction' }, initialize: function() { BaseView.prototype.initialize.apply(this, arguments); this.listenTo(AppState, 'change:graph_interval', this.render); var isAdmin = this.model.get('isAdmin'); this.model.fetch() .done(function(data) { this.template = require('./topic.hbs'); this.render({'message': data['message'], 'isAdmin': isAdmin}); }.bind(this)) .fail(this.handleViewError.bind(this)) .always(Pubsub.trigger.bind(Pubsub, 'view:ready')); }, topicAction: function(e) { e.preventDefault(); e.stopPropagation(); var action = $(e.currentTarget).data('action'); var txt = 'Are you sure you want to ' + action + ' ' + this.model.get('name') + '?'; bootbox.confirm(txt, function(result) { if (result !== true) { return; } if (action === 'delete') { $.ajax(this.model.url(), {'method': 'DELETE'}) .done(function() { window.location = AppState.basePath('/'); }); } else { $.post(this.model.url(), JSON.stringify({'action': action})) .done(function() { window.location.reload(true); }) .fail(this.handleAJAXError.bind(this)); } }.bind(this)); } }); module.exports = TopicView; ================================================ FILE: nsqadmin/static/js/views/topics.hbs ================================================ {{> warning}} {{> error}}

    Topics

    {{#if collection.length}} {{#if graph_active}}{{/if}} {{#if graph_active}}{{/if}} {{#if graph_active}}{{/if}} {{#each collection}} {{#if ../graph_active}}{{/if}} {{#if ../graph_active}}{{/if}} {{#if ../graph_active}}{{/if}} {{/each}}
    TopicDepthMessagesRate
    {{name}}
    {{else}}

    Notice

    No Topics Found
    {{/if}}
    ================================================ FILE: nsqadmin/static/js/views/topics.js ================================================ var Pubsub = require('../lib/pubsub'); var AppState = require('../app_state'); var BaseView = require('./base'); var Topics = require('../collections/topics'); var TopicsView = BaseView.extend({ className: 'topics container-fluid', template: require('./spinner.hbs'), initialize: function() { BaseView.prototype.initialize.apply(this, arguments); this.listenTo(AppState, 'change:graph_interval', this.render); this.collection = new Topics(); this.collection.fetch() .done(function(data) { this.template = require('./topics.hbs'); this.render({'message': data['message']}); }.bind(this)) .fail(this.handleViewError.bind(this)) .always(Pubsub.trigger.bind(Pubsub, 'view:ready')); } }); module.exports = TopicsView; ================================================ FILE: nsqadmin/static/js/views/warning.hbs ================================================
    {{message}}
    ================================================ FILE: nsqadmin/static.go ================================================ package nsqadmin import ( "embed" ) //go:embed static/build var static embed.FS func staticAsset(name string) ([]byte, error) { return static.ReadFile("static/build/" + name) } ================================================ FILE: nsqadmin/test/ca.key ================================================ -----BEGIN RSA PRIVATE KEY----- Proc-Type: 4,ENCRYPTED DEK-Info: DES-EDE3-CBC,D6208487450BDF1C tucU/4j6agGQlW60D8V3Zr/QHcLhhyFagres1gGdWfGIqluNZb7omki/XidHXJSG eB9vV2Xb12/8umc31e7Mnmn9hd34230v/KlAnJ4ukDpJpbmjnEx3F9uiqYFi/yxQ avSsfF6Tsh3XOh3Oe27I/xfYx37g6Agd+EQEJ1hvWvygMIJMTDMP5ZaFoZANtFLy hDEZ6woJSn9avF/L+1GW8jl2aI1QbdKkK0jDHgFAwUI4sjWeXvEQNNYY3trTIoMo wab3vi+4XziFONbS4OZrZUYfZPB5YOFbtT2whzggp2HdSTiu48/Ld3N8SjuMrKfm uR+nd+ovQ5kVWHInzWAIXSyPhgR9ZY8eyXaHNJJfzNu3HY72lfzD/NtZfacMRBr6 M3Wg/OKPS7ZrtqCWkY9P3KK9Cul8Jzy229fSqHo8Rg4= -----END RSA PRIVATE KEY----- ================================================ FILE: nsqadmin/test/ca.pem ================================================ -----BEGIN CERTIFICATE----- MIIBVDCB/6ADAgECAgkAvHG4Z/7nX/gwDQYJKoZIhvcNAQEFBQAwDTELMAkGA1UE AwwCY2EwHhcNMTcwOTE2MTc0NDE0WhcNMjcwOTE0MTc0NDE0WjANMQswCQYDVQQD DAJjYTBcMA0GCSqGSIb3DQEBAQUAA0sAMEgCQQDJAM3Tr1BoxlLJtTy0oRcp93dT 9hhHwms8P1V3k2FpXYRS4deUo+uwcAM9KGDt9VMXVBEchtI4VYTvLgatBPUBAgMB AAGjQjBAMA8GA1UdEwEB/wQFMAMBAf8wHQYDVR0OBBYEFMyLx7rjKBe/xZQLnVzI uqNNVzxRMA4GA1UdDwEB/wQEAwIBBjANBgkqhkiG9w0BAQUFAANBAJiN1XgPPlQ2 z7PhtzXStaz/BJVqhD7g9fsZsmoPX4ifDsTfzUsRB56Aq/NTsKiIYQkFPHH0donG ++a5ZVWjgYk= -----END CERTIFICATE----- ================================================ FILE: nsqadmin/test/ca.srl ================================================ 91418D04995922E7 ================================================ FILE: nsqadmin/test/cert.pem ================================================ -----BEGIN CERTIFICATE----- MIIEbjCCA1agAwIBAgIJAK6x7y6AwBmLMA0GCSqGSIb3DQEBBQUAMIGAMQswCQYD VQQGEwJVUzERMA8GA1UECBMITmV3IFlvcmsxFjAUBgNVBAcTDU5ldyBZb3JrIENp dHkxDDAKBgNVBAoTA05TUTETMBEGA1UEAxMKdGVzdC5sb2NhbDEjMCEGCSqGSIb3 DQEJARYUbXJlaWZlcnNvbkBnbWFpbC5jb20wHhcNMTMwNjI4MDA0MzQ4WhcNMTYw NDE3MDA0MzQ4WjCBgDELMAkGA1UEBhMCVVMxETAPBgNVBAgTCE5ldyBZb3JrMRYw FAYDVQQHEw1OZXcgWW9yayBDaXR5MQwwCgYDVQQKEwNOU1ExEzARBgNVBAMTCnRl c3QubG9jYWwxIzAhBgkqhkiG9w0BCQEWFG1yZWlmZXJzb25AZ21haWwuY29tMIIB IjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAnX0KB+svwy+yHU2qggz/EaGg craKShagKo+9M9y5HLM852ngk5c+t+tJJbx3N954Wr1FXBuGIv1ltU05rU4zhvBS 25tVP1UIEnT5pBt2TeetLkl199Y7fxh1hKmnwJMG3fy3VZdNXEndBombXMmtXpQY shuEJHKeUNDbQKz5X+GjEdkTPO/HY/VMHsxS23pbSimQozMg3hvLIdgv0aS3QECz ydZBgTPThy3uDtHIuCpxCwXd/vDF68ATlYgo3h3lh2vxNwM/pjklIUhzMh4XaKQF 7m3/0KbtUcXfy0QHueeuMr11E9MAFNyRN4xf9Fk1yB97KJ3PJBTC5WD/m1nW+QID AQABo4HoMIHlMB0GA1UdDgQWBBR3HMBws4lmYYSIgwoZsfW+bbgaMjCBtQYDVR0j BIGtMIGqgBR3HMBws4lmYYSIgwoZsfW+bbgaMqGBhqSBgzCBgDELMAkGA1UEBhMC VVMxETAPBgNVBAgTCE5ldyBZb3JrMRYwFAYDVQQHEw1OZXcgWW9yayBDaXR5MQww CgYDVQQKEwNOU1ExEzARBgNVBAMTCnRlc3QubG9jYWwxIzAhBgkqhkiG9w0BCQEW FG1yZWlmZXJzb25AZ21haWwuY29tggkArrHvLoDAGYswDAYDVR0TBAUwAwEB/zAN BgkqhkiG9w0BAQUFAAOCAQEANOYTbanW2iyV1v4oYpcM/y3TWcQKzSME8D2SGFZb dbMYU81hH3TTlQdvyeh3FAcdjhKE8Xi/RfNNjEslTBscdKXePGpZg6eXRNJzPP5K KZPf5u6tcpAeUOKrMqbGwbE+h2QixxG1EoVQtE421szsU2P7nHRTdHzKFRnOerfl Phm3NocR0P40Rv7WKdxpOvqc+XKf0onTruoVYoPWGpwcLixCG0zu4ZQ23/L/Dy18 4u70Hbq6O/6kq9FBFaDNp3IhiEdu2Cq6ZplU6bL9XDF27KIEErHwtuqBHVlMG+zB oH/k9vZvwH7OwAjHdKp+1yeZFLYC8K5hjFIHqcdwpZCNIg== -----END CERTIFICATE----- ================================================ FILE: nsqadmin/test/client.key ================================================ -----BEGIN RSA PRIVATE KEY----- MIIEowIBAAKCAQEAuMz5IJ01Wqaj2qJAU7o2aoc2iP1BRhTba+cWrsjbZA+6kXAk 19Dlz/nNK8gxsBMl3rzTGWXKUm/rKBDp680zJkj8nzcGApGlzbDoqXym8ixOniOC aH/RODPC8OtD2UozeXlUOQWK+ufkMvmVBc/IeJyFkcm7jhX+LHVFn+3iWLRNbzGm hbf8oNHvybS7HSuV3OYiHDJxA0d6IgozqVOYVSqdDabQ2FN/8nKzpMHjwMpfT008 5hUhr72E5XuJ3biPUwoOrRETSa1/EQcAEH0wSwHU1AGdnGh3/5ynVupfef0qkyqw NseYs9U++IUFXqA1IuEmAlJn5QczAHjPQe8xBQIDAQABAoIBAGx4CyZEgCOUOgrD P2SloPkIIk9n7x82cNA11I+E35kszkI9g7KVL77SDcZL/DYwFwNU68c1gvq+LFXZ D6RTTlmDb5v4TPPHD33a/8UzoD33GbIif5HcrC4D28FTJgDtV6dOOsw5X6kD4WK2 Me02V6HLpW677PVqHUV1FAfaNgf/1SIhJCechYSO+3pKPtli8DnsmOd336wNtWZR Q3Ctm63Eq7AiLmGLOkLYtll9Kr6WfCSFHBmqJg9MsQEfINXMCOB54Z+xkoijjJLO zNSk/3XvppB1JpmAusbo9Mq03Ci9JdinYSKxwVOG6e+5cosrOHpDQyZIWa87ZTpk kjLCgEECgYEA33JCGTEwaqfAuYOspCZvi2Mqz7t6QiwvwxgDf4JFLgDQArJfzKlC bAzaSU8bT+SN96CF9LU4XekN5/paGZAFjWxraPkK0oL28kQrtzfEQv2XL6GFoNAh WsQtoaGddL9VPWlCiHeueYERIDNrZCaVeAx6OV6W3CEwGOkRiQyntlUCgYEA07lf LpxF8aAgMToq3++D5RGAXaT2PYxhZZFb5Y2tS7a4T0ulQwXQi4H2dObGJ32zCjFG ls1H91mNKGBbyKnF1jT/dt/acqymvipASm1xq6uysIllA4xp56sDnJgG9bnXjYFY m2yVNIoIQGZ7jp7e6t1KQ9San6V8asWlvBLXX/ECgYEAuhF9dVj+xnH3DQTXSMIw 9NOZnO6zelMtWrqufwnN7ecDUJuVJupzw2JYi99yEO90QRbNNd+KlrkxuVFCojLK TOBR+VIZbv9cAJZACQxJRLfDpAhPLIDkpZ7jmMrqQYPqyX7TxqxTAB84UaY/8WAn 65YIWamo2ppQYQ4Eaim9pxkCgYAMwfW/TEFWruxhqvycY8VRzz0p51/DE6tmwFyG N4RCtK7kcE1z/Wy0i087ehBknslkCtYTDimQ+P9teGjvbXNzVdwy4Ig8MrUVblxT X8birkTlKFJC5XoYMJDWJb79nYYki6+4JdHTyaF3p/U4AdCy3ES2U6BBkGov0NsM uyHpMQKBgGc9wIGHwAGLg3iVHGXbXELdpzVztLRC7D12TPfdtGBLTD5NH5jAx8K/ w61+l4tzFcK3jfWzwdme277cBgfq/aw2DDt07vkOvPzhUwVZiE39bfddfIvyed99 XzEMf8THh5wdm6mLsOVgdfOfFcDi5ReIv1/7+eOd/MQi8TxIVfAu -----END RSA PRIVATE KEY----- ================================================ FILE: nsqadmin/test/client.pem ================================================ -----BEGIN CERTIFICATE----- MIICyzCCAnWgAwIBAgIJAJFBjQSZWSLnMA0GCSqGSIb3DQEBCwUAMA0xCzAJBgNV BAMMAmNhMB4XDTE3MDkxNjE3NDQxNFoXDTI3MDkxNDE3NDQxNFowgYcxIzAhBgkq hkiG9w0BCQEWFG1yZWlmZXJzb25AZ21haWwuY29tMQswCQYDVQQGEwJERTEMMAoG A1UECBMDTlJXMQ4wDAYDVQQHEwVFYXJ0aDEXMBUGA1UEChMOUmFuZG9tIENvbXBh bnkxCzAJBgNVBAsTAklUMQ8wDQYDVQQDEwZuc3EuaW8wggEiMA0GCSqGSIb3DQEB AQUAA4IBDwAwggEKAoIBAQC4zPkgnTVapqPaokBTujZqhzaI/UFGFNtr5xauyNtk D7qRcCTX0OXP+c0ryDGwEyXevNMZZcpSb+soEOnrzTMmSPyfNwYCkaXNsOipfKby LE6eI4Jof9E4M8Lw60PZSjN5eVQ5BYr65+Qy+ZUFz8h4nIWRybuOFf4sdUWf7eJY tE1vMaaFt/yg0e/JtLsdK5Xc5iIcMnEDR3oiCjOpU5hVKp0NptDYU3/ycrOkwePA yl9PTTzmFSGvvYTle4nduI9TCg6tERNJrX8RBwAQfTBLAdTUAZ2caHf/nKdW6l95 /SqTKrA2x5iz1T74hQVeoDUi4SYCUmflBzMAeM9B7zEFAgMBAAGjdTBzMAwGA1Ud EwEB/wQCMAAwHQYDVR0OBBYEFMzYXiTD7moi0oXqSQ5D2LsO51GXMB8GA1UdIwQY MBaAFMyLx7rjKBe/xZQLnVzIuqNNVzxRMA4GA1UdDwEB/wQEAwIFoDATBgNVHSUE DDAKBggrBgEFBQcDAjANBgkqhkiG9w0BAQsFAANBAAujxas6toRhMl/+kZrly0G/ AvjbA3WY5cLLIdffGdQ5bsS3aOP23nj98ut7unNUNsCo+eUwpJgvabnFnL+NFZA= -----END CERTIFICATE----- ================================================ FILE: nsqadmin/test/client.req ================================================ -----BEGIN CERTIFICATE REQUEST----- MIICzTCCAbUCAQAwgYcxIzAhBgkqhkiG9w0BCQEWFG1yZWlmZXJzb25AZ21haWwu Y29tMQswCQYDVQQGEwJERTEMMAoGA1UECBMDTlJXMQ4wDAYDVQQHEwVFYXJ0aDEX MBUGA1UEChMOUmFuZG9tIENvbXBhbnkxCzAJBgNVBAsTAklUMQ8wDQYDVQQDEwZu c3EuaW8wggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQC4zPkgnTVapqPa okBTujZqhzaI/UFGFNtr5xauyNtkD7qRcCTX0OXP+c0ryDGwEyXevNMZZcpSb+so EOnrzTMmSPyfNwYCkaXNsOipfKbyLE6eI4Jof9E4M8Lw60PZSjN5eVQ5BYr65+Qy +ZUFz8h4nIWRybuOFf4sdUWf7eJYtE1vMaaFt/yg0e/JtLsdK5Xc5iIcMnEDR3oi CjOpU5hVKp0NptDYU3/ycrOkwePAyl9PTTzmFSGvvYTle4nduI9TCg6tERNJrX8R BwAQfTBLAdTUAZ2caHf/nKdW6l95/SqTKrA2x5iz1T74hQVeoDUi4SYCUmflBzMA eM9B7zEFAgMBAAGgADANBgkqhkiG9w0BAQsFAAOCAQEALD1VjbBjaeX7l7JR4IuL wrHVrFDiRYsgvqyw39j2MC95VwrGwzf4cXCE+RuqE/DhbV9UI7sWKaJfFs9Usq+g VSoKnHSEylt34y6ABSc5eAik+GnheJZbJ6UDjxcvNd0UpFGMrHbsXVyQd0Y1XAu7 nxCYIa82kNA+Opb+ra03hkLC7wvRLbXTOB2g6JLkyhYR6S5GkOFTNnz2AJerN7zt NEL7owlRcjyIsL5tjDpDH1944NNtzhgmrUeIjB08reayuot9RKznMVGwBfY6DIHM Q4uNN3CMOOoAHr1UzvBf/qfvb6ltPTMKSV1OncLlC3C59NoO8vhKIHCN18Ya2OMu rw== -----END CERTIFICATE REQUEST----- ================================================ FILE: nsqadmin/test/key.pem ================================================ -----BEGIN RSA PRIVATE KEY----- MIIEogIBAAKCAQEAnX0KB+svwy+yHU2qggz/EaGgcraKShagKo+9M9y5HLM852ng k5c+t+tJJbx3N954Wr1FXBuGIv1ltU05rU4zhvBS25tVP1UIEnT5pBt2TeetLkl1 99Y7fxh1hKmnwJMG3fy3VZdNXEndBombXMmtXpQYshuEJHKeUNDbQKz5X+GjEdkT PO/HY/VMHsxS23pbSimQozMg3hvLIdgv0aS3QECzydZBgTPThy3uDtHIuCpxCwXd /vDF68ATlYgo3h3lh2vxNwM/pjklIUhzMh4XaKQF7m3/0KbtUcXfy0QHueeuMr11 E9MAFNyRN4xf9Fk1yB97KJ3PJBTC5WD/m1nW+QIDAQABAoIBACvtfKbIywG+hAf4 ad7skRjx5DcbA2e29+XnQfb9UgTXWd2SgrmoLi5OypBkCTzkKN3mfTo70yZfV8dC Sxwz+9tfnTz0DssjhKThS+CiaFVCkeOfSfBfKSlCQUVHrSrh18CDhP+yvDlJwQTZ zSQMfPcsh9bmJe2kqtQP7ZgUp1o+vaB8Sju8YYrO6FllxbdLRGm4pfvvrHIRRmXa oVHn0ei0JpwoTY9kHYht4LNeJnbP/MCWdmcuv3Gnel7jAlhaKab5aNIGr0Xe7aIQ iX6mpZ0/Rnt8o/XcTOg8l3ruIdVuySX6SYn08JMnfFkXdNYRVhoV1tC5ElWkaZLf hPmj2yECgYEAyts0R0b8cZ6HTAyuLm3ilw0s0v0/MM9ZtaqMRilr2WEtAhF0GpHG TzmGnii0WcTNXD7NTsNcECR/0ZpXPRleMczsL2Juwd4FkQ37h7hdKPseJNrfyHRg VolOFBX9H14C3wMB9cwdsG4Egw7fE27WCoreEquHgwFxl1zBrXKH088CgYEAxr8w BKZs0bF7LRrFT5pH8hpMLYHMYk8ZIOfgmEGVBKDQCOERPR9a9kqUss7wl/98LVNK RnFlyWD6Z0/QcQsLL4LjBeZJ25qEMc6JXm9VGAzhXA1ZkUofVoYCnG+f6KUn8CuJ /AcV2ZDFsEP10IiQG0hKsceXiwFEvEr8306tMrcCgYBLgnscSR0xAeyk71dq6vZc ecgEpcX+2kAvclOSzlpZ6WVCjtKkDT0/Qk+M0eQIQkybGLl9pxS+4Yc+s2/jy2yX pwsHvGE0AvwZeZX2eDcdSRR4bYy9ZixyKdwJeAHnyivRbaIuJ5Opl9pQGpoI9snv 1K9DTdw8dK4exKVHdgl/WwKBgDkmLsuXg4EEtPOyV/xc08VVNIR9Z2T5c7NXmeiO KyiKiWeUOF3ID2L07S9BfENozq9F3PzGjMtMXJSqibiHwW6nB1rh7mj8VHjx9+Q0 xVZGFeNfX1r84mgB3uxW2LeQDhzsmB/lda37CC14TU3qhu2hawEV8IijE73FHlOk Dv+fAoGAI4/XO5o5tNn5Djo8gHmGMCbinUE9+VySxl7wd7PK8w2VSofO88ofixDk NX94yBYhg5WZcLdPm45RyUnq+WVQYz9IKUrdxLFTH+wxyzUqZCW7jgXCvWV+071q vqm9C+kndq+18/1VKuCSGWnF7Ay4lbsgPXY2s4VKRxcb3QpZSPU= -----END RSA PRIVATE KEY----- ================================================ FILE: nsqadmin/test/server.key ================================================ -----BEGIN RSA PRIVATE KEY----- MIIEowIBAAKCAQEAuYs7NerYpXWq/IPaBGriHobhIs/i4AJHra7QhZ5N/EfBCb0N 6K3tI4GRHMZ4lB0HwINWmxaPs6D2yDCxpGgnru6qCu0/91+Lq+GEZj+Fz1E2yfCq tecTySvWf9en+SdTKFZTGGIAu3zFcozuX2fdhkjJ6ibZhnrXKlPU4IN0STkkRfFX +yb/6P88mGh7PZKBb67Qw1bppzXE/SWgHVqDqu++whT1quwxnRhh/SHgwugN+6iK uTDcJ1jQ1QWe+xna7Te6nNBUvL6dV49jpoDOjHsv+UaAvzNeHCHqypTC0wwYHXyg TTAU+E20c2NOCC7P0PAid1liNeuuEWtM2bRrFQIDAQABAoIBABPzc7d1fDw2bd9f Mic9cvkDWdwLbILX2+tCG+vyPMJ+2LP6Xy+A3DnwKbFlafvLL1U1Ci/8+hC/oymd isx54qJ9yU0Je9JWtMcTpc/0zqefPPvz4/dRVKBSFWuDve0dnGR++8poZ1nBrd2G Z+9cVMamtwd1i/hY5yAHCaHmoK9qxXuSjtbPkqDqruloGO5iT+tcICjAeWyJIx1P 5VJjuKDx4AhU4B8DpBSTtA02BKltvr2K6D4zhJ255oI4iLUkYk4IwxB7yZH+6Y++ mDa6e8iM3ut+qMkCxto/CfMCqYfoRzmykiazZ5qFXfGwv3OkWUEOBXU//Vpklvnh OQWYL/UCgYEA7mkKXbCrCNJEdly4LVLPKPFjVUPhMdXcqdnoKZ6BiVYFMqTuG4+w o/Rf6ZpqZ3Rf656/ypd1atVvK2g4zfq3j1W6lDnpLjKaLhc8a+ZH4NWXBORIzlUW aR2xVcPpAokor37FRAWbduiwVSCHlDtg48i+rvukyqmeRBKjrIgX3lsCgYEAxzut 5GiMZ9TkjqYXdlUDIDKWMHMfLfRY7+Q/cBdr7AfiMVw4PIWJG/IuxCGwGp77gRg6 BeBsi8Htj5/EGbRiK4kjFyf8LqYGOeYLZ3I8+olr+tNmF4/yKwxku3VylpmrpCAV /8emso1rAWo6Y98MTNSGgvLqhDU6OD7tTSv3908CgYA4+aVeio/1RbrSxonFWxri 3/0rLVOuAzv+43KWL6kpVwNa/QtiTs6aABbDzwFKxAcAWinfkp6e727n4rpgj2A6 wvQZ5FUTk0hBZ5ArAReAZcr3gk7b8H2wlUYCBxWyY3Dzr8oY3XYvzqAFWAbOp/oZ tanMS5swS6TlA8dVvhhmLQKBgGZwcBPOEctdcntKOSwVv/qxJ/oXZ0O4rHYENP4M fOgqkYnxsdSkkH/3AUbFT4gQkJ6q90KIRyeA+gXsDudskUFzTMCeRZMyuGbSurBg 06u6NvQL+CVLVSf/QlgEpnt63f8QpF8Up8iM4CUlGoq5Z9ilOdhg0GZT+/BpopgY cHIPAoGBAKyLumgeG+gMFK5x/Fi6zjBT5MK8Tw0VMkahW01Jx2laYibS/a8AEFmn ySdrmLPkOmmWgXCk3m2m5PkvM5qH/KNugOA0+WX2CTvwt4ZgwWUalQ5R43wSIeDC MXVfwC8uE66PmpYgmsu4H0vnCGfacOCQhfdq01SLobgBiQSrm/6D -----END RSA PRIVATE KEY----- ================================================ FILE: nsqadmin/test/server.pem ================================================ -----BEGIN CERTIFICATE----- MIIC3jCCAoigAwIBAgIJAJFBjQSZWSLmMA0GCSqGSIb3DQEBCwUAMA0xCzAJBgNV BAMMAmNhMB4XDTE3MDkxNjE3NDQxNFoXDTI3MDkxNDE3NDQxNFowgYcxIzAhBgkq hkiG9w0BCQEWFG1yZWlmZXJzb25AZ21haWwuY29tMQswCQYDVQQGEwJERTEMMAoG A1UECBMDTlJXMQ4wDAYDVQQHEwVFYXJ0aDEXMBUGA1UEChMOUmFuZG9tIENvbXBh bnkxCzAJBgNVBAsTAklUMQ8wDQYDVQQDEwZuc3EuaW8wggEiMA0GCSqGSIb3DQEB AQUAA4IBDwAwggEKAoIBAQC5izs16tildar8g9oEauIehuEiz+LgAketrtCFnk38 R8EJvQ3ore0jgZEcxniUHQfAg1abFo+zoPbIMLGkaCeu7qoK7T/3X4ur4YRmP4XP UTbJ8Kq15xPJK9Z/16f5J1MoVlMYYgC7fMVyjO5fZ92GSMnqJtmGetcqU9Tgg3RJ OSRF8Vf7Jv/o/zyYaHs9koFvrtDDVumnNcT9JaAdWoOq777CFPWq7DGdGGH9IeDC 6A37qIq5MNwnWNDVBZ77GdrtN7qc0FS8vp1Xj2OmgM6Mey/5RoC/M14cIerKlMLT DBgdfKBNMBT4TbRzY04ILs/Q8CJ3WWI1664Ra0zZtGsVAgMBAAGjgYcwgYQwDAYD VR0TAQH/BAIwADAdBgNVHQ4EFgQUt6UXJ8BQB++I/cQG5ZgaiUyqOE8wHwYDVR0j BBgwFoAUzIvHuuMoF7/FlAudXMi6o01XPFEwDgYDVR0PAQH/BAQDAgWgMBMGA1Ud JQQMMAoGCCsGAQUFBwMBMA8GA1UdEQQIMAaHBH8AAAEwDQYJKoZIhvcNAQELBQAD QQCCwm0F2eTmtIcKDxrXGZ7q9y9mfsROfYaCnH+vKUDw2vmqQkInzhLeEElDGQcR ww0IKCnDHEruNb2tKyQM/70L -----END CERTIFICATE----- ================================================ FILE: nsqadmin/test/server.req ================================================ -----BEGIN CERTIFICATE REQUEST----- MIICzTCCAbUCAQAwgYcxIzAhBgkqhkiG9w0BCQEWFG1yZWlmZXJzb25AZ21haWwu Y29tMQswCQYDVQQGEwJERTEMMAoGA1UECBMDTlJXMQ4wDAYDVQQHEwVFYXJ0aDEX MBUGA1UEChMOUmFuZG9tIENvbXBhbnkxCzAJBgNVBAsTAklUMQ8wDQYDVQQDEwZu c3EuaW8wggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQC5izs16tildar8 g9oEauIehuEiz+LgAketrtCFnk38R8EJvQ3ore0jgZEcxniUHQfAg1abFo+zoPbI MLGkaCeu7qoK7T/3X4ur4YRmP4XPUTbJ8Kq15xPJK9Z/16f5J1MoVlMYYgC7fMVy jO5fZ92GSMnqJtmGetcqU9Tgg3RJOSRF8Vf7Jv/o/zyYaHs9koFvrtDDVumnNcT9 JaAdWoOq777CFPWq7DGdGGH9IeDC6A37qIq5MNwnWNDVBZ77GdrtN7qc0FS8vp1X j2OmgM6Mey/5RoC/M14cIerKlMLTDBgdfKBNMBT4TbRzY04ILs/Q8CJ3WWI1664R a0zZtGsVAgMBAAGgADANBgkqhkiG9w0BAQsFAAOCAQEASNjcoZbyNcwMQjMmNoil S/7pCRn4aYzZIVjrVtOHQ9GHC23MSep5um2gIcMFPiuYyu9Byl8CSVtc1op2fAKS vrugoZaCrp/A76hqOfNxgh7VmgTux8bG5Qcjaija1BNWpbyaWARdBxN/WgS5CpCj u2yzv8mrzzFNrDMlsmiEMvtkMzdhiZ4YY8zm6CdrbIR5z1eqf4e+rs4oJtTKNNAD hewk8CGiUW1hOx2jpjcIVMRy+ofVHRX2xQ6Sw8qxCNsiv8IPAAivgAbFJO76ZSbH eQ7uKWszmBEroyFvZ0rfmFLXuopU125pyBDl5FUKYAZzCBx9tr5dROCbw/rXDhke ig== -----END CERTIFICATE REQUEST----- ================================================ FILE: nsqd/README.md ================================================ ## nsqd `nsqd` is the daemon that receives, queues, and delivers messages to clients. Read the [docs](https://nsq.io/components/nsqd.html) ================================================ FILE: nsqd/backend_queue.go ================================================ package nsqd // BackendQueue represents the behavior for the secondary message // storage system type BackendQueue interface { Put([]byte) error ReadChan() <-chan []byte // this is expected to be an *unbuffered* channel Close() error Delete() error Depth() int64 Empty() error } ================================================ FILE: nsqd/buffer_pool.go ================================================ package nsqd import ( "bytes" "sync" ) var bp sync.Pool func init() { bp.New = func() interface{} { return &bytes.Buffer{} } } func bufferPoolGet() *bytes.Buffer { return bp.Get().(*bytes.Buffer) } func bufferPoolPut(b *bytes.Buffer) { b.Reset() bp.Put(b) } ================================================ FILE: nsqd/channel.go ================================================ package nsqd import ( "container/heap" "errors" "fmt" "math" "strings" "sync" "sync/atomic" "time" "github.com/nsqio/go-diskqueue" "github.com/nsqio/nsq/internal/lg" "github.com/nsqio/nsq/internal/pqueue" "github.com/nsqio/nsq/internal/quantile" ) type Consumer interface { UnPause() Pause() Close() error TimedOutMessage() Stats(string) ClientStats Empty() } // Channel represents the concrete type for a NSQ channel (and also // implements the Queue interface) // // There can be multiple channels per topic, each with there own unique set // of subscribers (clients). // // Channels maintain all client and message metadata, orchestrating in-flight // messages, timeouts, requeuing, etc. type Channel struct { // 64bit atomic vars need to be first for proper alignment on 32bit platforms requeueCount uint64 messageCount uint64 zoneLocalMsgCount uint64 regionLocalMsgCount uint64 globalMsgCount uint64 timeoutCount uint64 sync.RWMutex topicName string name string nsqd *NSQD backend BackendQueue topologyAwareConsumption bool zoneLocalMsgChan chan *Message regionLocalMsgChan chan *Message memoryMsgChan chan *Message exitFlag int32 exitMutex sync.RWMutex // state tracking clients map[int64]Consumer paused int32 ephemeral bool deleteCallback func(*Channel) deleter sync.Once // Stats tracking e2eProcessingLatencyStream *quantile.Quantile // TODO: these can be DRYd up deferredMessages map[MessageID]*pqueue.Item deferredPQ pqueue.PriorityQueue deferredMutex sync.Mutex inFlightMessages map[MessageID]*Message inFlightPQ inFlightPqueue inFlightMutex sync.Mutex } // NewChannel creates a new instance of the Channel type and returns a pointer func NewChannel(topicName string, channelName string, nsqd *NSQD, deleteCallback func(*Channel)) *Channel { c := &Channel{ topicName: topicName, name: channelName, memoryMsgChan: nil, clients: make(map[int64]Consumer), deleteCallback: deleteCallback, nsqd: nsqd, ephemeral: strings.HasSuffix(channelName, "#ephemeral"), topologyAwareConsumption: nsqd.getOpts().HasExperiment(TopologyAwareConsumption), } if nsqd.getOpts().TopologyRegion != "" { c.regionLocalMsgChan = make(chan *Message) } if nsqd.getOpts().TopologyZone != "" { c.zoneLocalMsgChan = make(chan *Message) } // avoid mem-queue if size == 0 for more consistent ordering if nsqd.getOpts().MemQueueSize > 0 || c.ephemeral { c.memoryMsgChan = make(chan *Message, nsqd.getOpts().MemQueueSize) } if len(nsqd.getOpts().E2EProcessingLatencyPercentiles) > 0 { c.e2eProcessingLatencyStream = quantile.New( nsqd.getOpts().E2EProcessingLatencyWindowTime, nsqd.getOpts().E2EProcessingLatencyPercentiles, ) } c.initPQ() if c.ephemeral { c.backend = newDummyBackendQueue() } else { dqLogf := func(level diskqueue.LogLevel, f string, args ...interface{}) { opts := nsqd.getOpts() lg.Logf(opts.Logger, opts.LogLevel, lg.LogLevel(level), f, args...) } // backend names, for uniqueness, automatically include the topic... backendName := getBackendName(topicName, channelName) c.backend = diskqueue.New( backendName, nsqd.getOpts().DataPath, nsqd.getOpts().MaxBytesPerFile, int32(minValidMsgLength), int32(nsqd.getOpts().MaxMsgSize)+minValidMsgLength, nsqd.getOpts().SyncEvery, nsqd.getOpts().SyncTimeout, dqLogf, ) } c.nsqd.Notify(c, !c.ephemeral) return c } func (c *Channel) initPQ() { pqSize := int(math.Max(1, float64(c.nsqd.getOpts().MemQueueSize)/10)) c.inFlightMutex.Lock() c.inFlightMessages = make(map[MessageID]*Message) c.inFlightPQ = newInFlightPqueue(pqSize) c.inFlightMutex.Unlock() c.deferredMutex.Lock() c.deferredMessages = make(map[MessageID]*pqueue.Item) c.deferredPQ = pqueue.New(pqSize) c.deferredMutex.Unlock() } // Exiting returns a boolean indicating if this channel is closed/exiting func (c *Channel) Exiting() bool { return atomic.LoadInt32(&c.exitFlag) == 1 } // Delete empties the channel and closes func (c *Channel) Delete() error { return c.exit(true) } // Close cleanly closes the Channel func (c *Channel) Close() error { return c.exit(false) } func (c *Channel) exit(deleted bool) error { c.exitMutex.Lock() defer c.exitMutex.Unlock() if !atomic.CompareAndSwapInt32(&c.exitFlag, 0, 1) { return errors.New("exiting") } if deleted { c.nsqd.logf(LOG_INFO, "CHANNEL(%s): deleting", c.name) // since we are explicitly deleting a channel (not just at system exit time) // de-register this from the lookupd c.nsqd.Notify(c, !c.ephemeral) } else { c.nsqd.logf(LOG_INFO, "CHANNEL(%s): closing", c.name) } // this forceably closes client connections c.RLock() for _, client := range c.clients { client.Close() } c.RUnlock() if deleted { // empty the queue (deletes the backend files, too) c.Empty() return c.backend.Delete() } // write anything leftover to disk c.flush() return c.backend.Close() } func (c *Channel) Empty() error { c.Lock() defer c.Unlock() c.initPQ() for _, client := range c.clients { client.Empty() } for { select { case <-c.zoneLocalMsgChan: case <-c.regionLocalMsgChan: case <-c.memoryMsgChan: default: goto finish } } finish: return c.backend.Empty() } // flush persists all the messages in internal memory buffers to the backend // it does not drain inflight/deferred because it is only called in Close() func (c *Channel) flush() error { if len(c.zoneLocalMsgChan) > 0 || len(c.regionLocalMsgChan) > 0 || len(c.memoryMsgChan) > 0 || len(c.inFlightMessages) > 0 || len(c.deferredMessages) > 0 { c.nsqd.logf(LOG_INFO, "CHANNEL(%s): flushing %d memory %d in-flight %d deferred messages to backend", c.name, len(c.memoryMsgChan)+len(c.zoneLocalMsgChan)+len(c.regionLocalMsgChan), len(c.inFlightMessages), len(c.deferredMessages)) } for { select { case msg := <-c.zoneLocalMsgChan: err := writeMessageToBackend(msg, c.backend) if err != nil { c.nsqd.logf(LOG_ERROR, "failed to write message to backend - %s", err) } case msg := <-c.regionLocalMsgChan: err := writeMessageToBackend(msg, c.backend) if err != nil { c.nsqd.logf(LOG_ERROR, "failed to write message to backend - %s", err) } case msg := <-c.memoryMsgChan: err := writeMessageToBackend(msg, c.backend) if err != nil { c.nsqd.logf(LOG_ERROR, "failed to write message to backend - %s", err) } default: goto finish } } finish: c.inFlightMutex.Lock() for _, msg := range c.inFlightMessages { err := writeMessageToBackend(msg, c.backend) if err != nil { c.nsqd.logf(LOG_ERROR, "failed to write message to backend - %s", err) } } c.inFlightMutex.Unlock() c.deferredMutex.Lock() for _, item := range c.deferredMessages { msg := item.Value.(*Message) err := writeMessageToBackend(msg, c.backend) if err != nil { c.nsqd.logf(LOG_ERROR, "failed to write message to backend - %s", err) } } c.deferredMutex.Unlock() return nil } func (c *Channel) Depth() int64 { return int64(len(c.memoryMsgChan)) + int64(len(c.zoneLocalMsgChan)) + int64(len(c.regionLocalMsgChan)) + c.backend.Depth() } func (c *Channel) Pause() error { return c.doPause(true) } func (c *Channel) UnPause() error { return c.doPause(false) } func (c *Channel) doPause(pause bool) error { if pause { atomic.StoreInt32(&c.paused, 1) } else { atomic.StoreInt32(&c.paused, 0) } c.RLock() for _, client := range c.clients { if pause { client.Pause() } else { client.UnPause() } } c.RUnlock() return nil } func (c *Channel) IsPaused() bool { return atomic.LoadInt32(&c.paused) == 1 } // PutMessage writes a Message to the queue func (c *Channel) PutMessage(m *Message) error { c.exitMutex.RLock() defer c.exitMutex.RUnlock() if c.Exiting() { return errors.New("exiting") } err := c.put(m) if err != nil { return err } atomic.AddUint64(&c.messageCount, 1) return nil } func (c *Channel) put(m *Message) error { if c.topologyAwareConsumption { // Attempt zone local, region local and finally the memory channel // we do this to ensure that we preferentially deliver messages based on toplogy // // Because messagePump is intermittently unavailable while writing a msg to a client // we continue to add lower priority channels in the select loop, this means at each // attempt a higher priority channel can still win select { case c.zoneLocalMsgChan <- m: return nil default: } select { case c.zoneLocalMsgChan <- m: return nil case c.regionLocalMsgChan <- m: return nil default: } select { case c.zoneLocalMsgChan <- m: return nil case c.regionLocalMsgChan <- m: return nil case c.memoryMsgChan <- m: return nil default: } } else { select { case c.memoryMsgChan <- m: return nil default: } } err := writeMessageToBackend(m, c.backend) c.nsqd.SetHealth(err) if err != nil { c.nsqd.logf(LOG_ERROR, "CHANNEL(%s): failed to write message to backend - %s", c.name, err) return err } return nil } func (c *Channel) PutMessageDeferred(msg *Message, timeout time.Duration) { atomic.AddUint64(&c.messageCount, 1) c.StartDeferredTimeout(msg, timeout) } // TouchMessage resets the timeout for an in-flight message func (c *Channel) TouchMessage(clientID int64, id MessageID, clientMsgTimeout time.Duration) error { msg, err := c.popInFlightMessage(clientID, id) if err != nil { return err } c.removeFromInFlightPQ(msg) newTimeout := time.Now().Add(clientMsgTimeout) if newTimeout.Sub(msg.deliveryTS) >= c.nsqd.getOpts().MaxMsgTimeout { // we would have gone over, set to the max newTimeout = msg.deliveryTS.Add(c.nsqd.getOpts().MaxMsgTimeout) } msg.pri = newTimeout.UnixNano() err = c.pushInFlightMessage(msg) if err != nil { return err } c.addToInFlightPQ(msg) return nil } // FinishMessage successfully discards an in-flight message func (c *Channel) FinishMessage(clientID int64, id MessageID) error { msg, err := c.popInFlightMessage(clientID, id) if err != nil { return err } c.removeFromInFlightPQ(msg) if c.e2eProcessingLatencyStream != nil { c.e2eProcessingLatencyStream.Insert(msg.Timestamp) } return nil } // RequeueMessage requeues a message based on `time.Duration`, ie: // // `timeoutMs` == 0 - requeue a message immediately // `timeoutMs` > 0 - asynchronously wait for the specified timeout // // and requeue a message (aka "deferred requeue") func (c *Channel) RequeueMessage(clientID int64, id MessageID, timeout time.Duration) error { // remove from inflight first msg, err := c.popInFlightMessage(clientID, id) if err != nil { return err } c.removeFromInFlightPQ(msg) atomic.AddUint64(&c.requeueCount, 1) if timeout == 0 { c.exitMutex.RLock() if c.Exiting() { c.exitMutex.RUnlock() return errors.New("exiting") } err := c.put(msg) c.exitMutex.RUnlock() return err } // deferred requeue return c.StartDeferredTimeout(msg, timeout) } // AddClient adds a client to the Channel's client list func (c *Channel) AddClient(clientID int64, client Consumer) error { c.exitMutex.RLock() defer c.exitMutex.RUnlock() if c.Exiting() { return errors.New("exiting") } c.RLock() _, ok := c.clients[clientID] numClients := len(c.clients) c.RUnlock() if ok { return nil } maxChannelConsumers := c.nsqd.getOpts().MaxChannelConsumers if maxChannelConsumers != 0 && numClients >= maxChannelConsumers { return fmt.Errorf("consumers for %s:%s exceeds limit of %d", c.topicName, c.name, maxChannelConsumers) } c.Lock() c.clients[clientID] = client c.Unlock() return nil } // RemoveClient removes a client from the Channel's client list func (c *Channel) RemoveClient(clientID int64) { c.exitMutex.RLock() defer c.exitMutex.RUnlock() if c.Exiting() { return } c.RLock() _, ok := c.clients[clientID] c.RUnlock() if !ok { return } c.Lock() delete(c.clients, clientID) numClients := len(c.clients) c.Unlock() if numClients == 0 && c.ephemeral { go c.deleter.Do(func() { c.deleteCallback(c) }) } } func (c *Channel) StartInFlightTimeout(msg *Message, clientID int64, timeout time.Duration) error { now := time.Now() msg.clientID = clientID msg.deliveryTS = now msg.pri = now.Add(timeout).UnixNano() err := c.pushInFlightMessage(msg) if err != nil { return err } c.addToInFlightPQ(msg) return nil } func (c *Channel) StartDeferredTimeout(msg *Message, timeout time.Duration) error { absTs := time.Now().Add(timeout).UnixNano() item := &pqueue.Item{Value: msg, Priority: absTs} err := c.pushDeferredMessage(item) if err != nil { return err } c.addToDeferredPQ(item) return nil } // pushInFlightMessage atomically adds a message to the in-flight dictionary func (c *Channel) pushInFlightMessage(msg *Message) error { c.inFlightMutex.Lock() _, ok := c.inFlightMessages[msg.ID] if ok { c.inFlightMutex.Unlock() return errors.New("ID already in flight") } c.inFlightMessages[msg.ID] = msg c.inFlightMutex.Unlock() return nil } // popInFlightMessage atomically removes a message from the in-flight dictionary func (c *Channel) popInFlightMessage(clientID int64, id MessageID) (*Message, error) { c.inFlightMutex.Lock() msg, ok := c.inFlightMessages[id] if !ok { c.inFlightMutex.Unlock() return nil, errors.New("ID not in flight") } if msg.clientID != clientID { c.inFlightMutex.Unlock() return nil, errors.New("client does not own message") } delete(c.inFlightMessages, id) c.inFlightMutex.Unlock() return msg, nil } func (c *Channel) addToInFlightPQ(msg *Message) { c.inFlightMutex.Lock() c.inFlightPQ.Push(msg) c.inFlightMutex.Unlock() } func (c *Channel) removeFromInFlightPQ(msg *Message) { c.inFlightMutex.Lock() if msg.index == -1 { // this item has already been popped off the pqueue c.inFlightMutex.Unlock() return } c.inFlightPQ.Remove(msg.index) c.inFlightMutex.Unlock() } func (c *Channel) pushDeferredMessage(item *pqueue.Item) error { c.deferredMutex.Lock() // TODO: these map lookups are costly id := item.Value.(*Message).ID _, ok := c.deferredMessages[id] if ok { c.deferredMutex.Unlock() return errors.New("ID already deferred") } c.deferredMessages[id] = item c.deferredMutex.Unlock() return nil } func (c *Channel) popDeferredMessage(id MessageID) (*pqueue.Item, error) { c.deferredMutex.Lock() // TODO: these map lookups are costly item, ok := c.deferredMessages[id] if !ok { c.deferredMutex.Unlock() return nil, errors.New("ID not deferred") } delete(c.deferredMessages, id) c.deferredMutex.Unlock() return item, nil } func (c *Channel) addToDeferredPQ(item *pqueue.Item) { c.deferredMutex.Lock() heap.Push(&c.deferredPQ, item) c.deferredMutex.Unlock() } func (c *Channel) processDeferredQueue(t int64) bool { c.exitMutex.RLock() defer c.exitMutex.RUnlock() if c.Exiting() { return false } dirty := false for { c.deferredMutex.Lock() item, _ := c.deferredPQ.PeekAndShift(t) c.deferredMutex.Unlock() if item == nil { goto exit } dirty = true msg := item.Value.(*Message) _, err := c.popDeferredMessage(msg.ID) if err != nil { goto exit } c.put(msg) } exit: return dirty } func (c *Channel) processInFlightQueue(t int64) bool { c.exitMutex.RLock() defer c.exitMutex.RUnlock() if c.Exiting() { return false } dirty := false for { c.inFlightMutex.Lock() msg, _ := c.inFlightPQ.PeekAndShift(t) c.inFlightMutex.Unlock() if msg == nil { goto exit } dirty = true _, err := c.popInFlightMessage(msg.clientID, msg.ID) if err != nil { goto exit } atomic.AddUint64(&c.timeoutCount, 1) c.RLock() client, ok := c.clients[msg.clientID] c.RUnlock() if ok { client.TimedOutMessage() } c.put(msg) } exit: return dirty } ================================================ FILE: nsqd/channel_test.go ================================================ package nsqd import ( "fmt" "io" "net/http" "os" "strconv" "testing" "time" "github.com/nsqio/nsq/internal/test" ) // ensure that we can push a message through a topic and get it out of a channel func TestPutMessage(t *testing.T) { opts := NewOptions() opts.Logger = test.NewTestLogger(t) _, _, nsqd := mustStartNSQD(opts) defer os.RemoveAll(opts.DataPath) defer nsqd.Exit() topicName := "test_put_message" + strconv.Itoa(int(time.Now().Unix())) topic := nsqd.GetTopic(topicName) channel1 := topic.GetChannel("ch") var id MessageID msg := NewMessage(id, []byte("test")) topic.PutMessage(msg) outputMsg := <-channel1.memoryMsgChan test.Equal(t, msg.ID, outputMsg.ID) test.Equal(t, msg.Body, outputMsg.Body) } // ensure that both channels get the same message func TestPutMessage2Chan(t *testing.T) { opts := NewOptions() opts.Logger = test.NewTestLogger(t) _, _, nsqd := mustStartNSQD(opts) defer os.RemoveAll(opts.DataPath) defer nsqd.Exit() topicName := "test_put_message_2chan" + strconv.Itoa(int(time.Now().Unix())) topic := nsqd.GetTopic(topicName) channel1 := topic.GetChannel("ch1") channel2 := topic.GetChannel("ch2") var id MessageID msg := NewMessage(id, []byte("test")) topic.PutMessage(msg) outputMsg1 := <-channel1.memoryMsgChan test.Equal(t, msg.ID, outputMsg1.ID) test.Equal(t, msg.Body, outputMsg1.Body) outputMsg2 := <-channel2.memoryMsgChan test.Equal(t, msg.ID, outputMsg2.ID) test.Equal(t, msg.Body, outputMsg2.Body) } func TestInFlightWorker(t *testing.T) { count := 250 opts := NewOptions() opts.Logger = test.NewTestLogger(t) opts.MsgTimeout = 100 * time.Millisecond opts.QueueScanRefreshInterval = 100 * time.Millisecond _, _, nsqd := mustStartNSQD(opts) defer os.RemoveAll(opts.DataPath) defer nsqd.Exit() topicName := "test_in_flight_worker" + strconv.Itoa(int(time.Now().Unix())) topic := nsqd.GetTopic(topicName) channel := topic.GetChannel("channel") for i := 0; i < count; i++ { msg := NewMessage(topic.GenerateID(), []byte("test")) channel.StartInFlightTimeout(msg, 0, opts.MsgTimeout) } channel.Lock() inFlightMsgs := len(channel.inFlightMessages) channel.Unlock() test.Equal(t, count, inFlightMsgs) channel.inFlightMutex.Lock() inFlightPQMsgs := len(channel.inFlightPQ) channel.inFlightMutex.Unlock() test.Equal(t, count, inFlightPQMsgs) // the in flight worker has a resolution of 100ms so we need to wait // at least that much longer than our msgTimeout (in worst case) time.Sleep(4 * opts.MsgTimeout) channel.Lock() inFlightMsgs = len(channel.inFlightMessages) channel.Unlock() test.Equal(t, 0, inFlightMsgs) channel.inFlightMutex.Lock() inFlightPQMsgs = len(channel.inFlightPQ) channel.inFlightMutex.Unlock() test.Equal(t, 0, inFlightPQMsgs) } func TestChannelEmpty(t *testing.T) { opts := NewOptions() opts.Logger = test.NewTestLogger(t) _, _, nsqd := mustStartNSQD(opts) defer os.RemoveAll(opts.DataPath) defer nsqd.Exit() topicName := "test_channel_empty" + strconv.Itoa(int(time.Now().Unix())) topic := nsqd.GetTopic(topicName) channel := topic.GetChannel("channel") msgs := make([]*Message, 0, 25) for i := 0; i < 25; i++ { msg := NewMessage(topic.GenerateID(), []byte("test")) channel.StartInFlightTimeout(msg, 0, opts.MsgTimeout) msgs = append(msgs, msg) } channel.RequeueMessage(0, msgs[len(msgs)-1].ID, 100*time.Millisecond) test.Equal(t, 24, len(channel.inFlightMessages)) test.Equal(t, 24, len(channel.inFlightPQ)) test.Equal(t, 1, len(channel.deferredMessages)) test.Equal(t, 1, len(channel.deferredPQ)) channel.Empty() test.Equal(t, 0, len(channel.inFlightMessages)) test.Equal(t, 0, len(channel.inFlightPQ)) test.Equal(t, 0, len(channel.deferredMessages)) test.Equal(t, 0, len(channel.deferredPQ)) test.Equal(t, int64(0), channel.Depth()) } func TestChannelEmptyConsumer(t *testing.T) { opts := NewOptions() opts.Logger = test.NewTestLogger(t) tcpAddr, _, nsqd := mustStartNSQD(opts) defer os.RemoveAll(opts.DataPath) defer nsqd.Exit() conn, _ := mustConnectNSQD(tcpAddr) defer conn.Close() topicName := "test_channel_empty" + strconv.Itoa(int(time.Now().Unix())) topic := nsqd.GetTopic(topicName) channel := topic.GetChannel("channel") client := newClientV2(0, conn, nsqd) client.SetReadyCount(25) err := channel.AddClient(client.ID, client) test.Equal(t, err, nil) for i := 0; i < 25; i++ { msg := NewMessage(topic.GenerateID(), []byte("test")) channel.StartInFlightTimeout(msg, 0, opts.MsgTimeout) client.SendingMessage() } for _, cl := range channel.clients { stats := cl.Stats("").(ClientV2Stats) test.Equal(t, int64(25), stats.InFlightCount) } channel.Empty() for _, cl := range channel.clients { stats := cl.Stats("").(ClientV2Stats) test.Equal(t, int64(0), stats.InFlightCount) } } func TestMaxChannelConsumers(t *testing.T) { opts := NewOptions() opts.Logger = test.NewTestLogger(t) opts.MaxChannelConsumers = 1 tcpAddr, _, nsqd := mustStartNSQD(opts) defer os.RemoveAll(opts.DataPath) defer nsqd.Exit() conn, _ := mustConnectNSQD(tcpAddr) defer conn.Close() topicName := "test_max_channel_consumers" + strconv.Itoa(int(time.Now().Unix())) topic := nsqd.GetTopic(topicName) channel := topic.GetChannel("channel") client1 := newClientV2(1, conn, nsqd) client1.SetReadyCount(25) err := channel.AddClient(client1.ID, client1) test.Equal(t, err, nil) client2 := newClientV2(2, conn, nsqd) client2.SetReadyCount(25) err = channel.AddClient(client2.ID, client2) test.NotEqual(t, err, nil) } func TestChannelHealth(t *testing.T) { opts := NewOptions() opts.Logger = test.NewTestLogger(t) opts.MemQueueSize = 2 _, httpAddr, nsqd := mustStartNSQD(opts) defer os.RemoveAll(opts.DataPath) defer nsqd.Exit() topic := nsqd.GetTopic("test") channel := topic.GetChannel("channel") channel.backend = &errorBackendQueue{} msg := NewMessage(topic.GenerateID(), make([]byte, 100)) err := channel.PutMessage(msg) test.Nil(t, err) msg = NewMessage(topic.GenerateID(), make([]byte, 100)) err = channel.PutMessage(msg) test.Nil(t, err) msg = NewMessage(topic.GenerateID(), make([]byte, 100)) err = channel.PutMessage(msg) test.NotNil(t, err) url := fmt.Sprintf("http://%s/ping", httpAddr) resp, err := http.Get(url) test.Nil(t, err) test.Equal(t, 500, resp.StatusCode) body, _ := io.ReadAll(resp.Body) resp.Body.Close() test.Equal(t, "NOK - never gonna happen", string(body)) channel.backend = &errorRecoveredBackendQueue{} msg = NewMessage(topic.GenerateID(), make([]byte, 100)) err = channel.PutMessage(msg) test.Nil(t, err) resp, err = http.Get(url) test.Nil(t, err) test.Equal(t, 200, resp.StatusCode) body, _ = io.ReadAll(resp.Body) resp.Body.Close() test.Equal(t, "OK", string(body)) } ================================================ FILE: nsqd/client_v2.go ================================================ package nsqd import ( "bufio" "compress/flate" "crypto/tls" "fmt" "net" "strings" "sync" "sync/atomic" "time" "github.com/golang/snappy" "github.com/nsqio/nsq/internal/auth" ) const defaultBufferSize = 16 * 1024 const ( stateInit = iota stateDisconnected stateConnected stateSubscribed stateClosing ) type identifyDataV2 struct { ClientID string `json:"client_id"` Hostname string `json:"hostname"` HeartbeatInterval int `json:"heartbeat_interval"` OutputBufferSize int `json:"output_buffer_size"` OutputBufferTimeout int `json:"output_buffer_timeout"` FeatureNegotiation bool `json:"feature_negotiation"` TLSv1 bool `json:"tls_v1"` Deflate bool `json:"deflate"` DeflateLevel int `json:"deflate_level"` Snappy bool `json:"snappy"` SampleRate int32 `json:"sample_rate"` UserAgent string `json:"user_agent"` MsgTimeout int `json:"msg_timeout"` TopologyRegion string `json:"topology_region"` TopologyZone string `json:"topology_zone"` } type identifyEvent struct { OutputBufferTimeout time.Duration HeartbeatInterval time.Duration SampleRate int32 MsgTimeout time.Duration TopologyRegion string TopologyZone string } type PubCount struct { Topic string `json:"topic"` Count uint64 `json:"count"` } type ClientV2Stats struct { ClientID string `json:"client_id"` Hostname string `json:"hostname"` Version string `json:"version"` RemoteAddress string `json:"remote_address"` State int32 `json:"state"` ReadyCount int64 `json:"ready_count"` InFlightCount int64 `json:"in_flight_count"` MessageCount uint64 `json:"message_count"` ZoneLocalMsgCount uint64 `json:"zone_local_msg_count,omitempty"` RegionLocalMsgCount uint64 `json:"region_local_msg_count,omitempty"` GlobalMsgCount uint64 `json:"global_msg_count,omitempty"` FinishCount uint64 `json:"finish_count"` RequeueCount uint64 `json:"requeue_count"` ConnectTime int64 `json:"connect_ts"` SampleRate int32 `json:"sample_rate"` Deflate bool `json:"deflate"` Snappy bool `json:"snappy"` UserAgent string `json:"user_agent"` Authed bool `json:"authed,omitempty"` AuthIdentity string `json:"auth_identity,omitempty"` AuthIdentityURL string `json:"auth_identity_url,omitempty"` TopologyZone string `json:"topology_zone"` TopologyRegion string `json:"topology_region"` PubCounts []PubCount `json:"pub_counts,omitempty"` TLS bool `json:"tls"` CipherSuite string `json:"tls_cipher_suite"` TLSVersion string `json:"tls_version"` TLSNegotiatedProtocol string `json:"tls_negotiated_protocol"` TLSNegotiatedProtocolIsMutual bool `json:"tls_negotiated_protocol_is_mutual"` } func (s ClientV2Stats) String() string { connectTime := time.Unix(s.ConnectTime, 0) duration := time.Since(connectTime).Truncate(time.Second) _, port, _ := net.SplitHostPort(s.RemoteAddress) id := fmt.Sprintf("%s:%s %s", s.Hostname, port, s.UserAgent) // producer if len(s.PubCounts) > 0 { var total uint64 var topicOut []string for _, v := range s.PubCounts { total += v.Count topicOut = append(topicOut, fmt.Sprintf("%s=%d", v.Topic, v.Count)) } return fmt.Sprintf("[%s %-21s] msgs: %-8d topics: %s connected: %s", s.Version, id, total, strings.Join(topicOut, ","), duration, ) } // consumer return fmt.Sprintf("[%s %-21s] state: %d inflt: %-4d rdy: %-4d fin: %-8d re-q: %-8d msgs: %-8d connected: %s", s.Version, id, s.State, s.InFlightCount, s.ReadyCount, s.FinishCount, s.RequeueCount, s.MessageCount, duration, ) } type clientV2 struct { // 64bit atomic vars need to be first for proper alignment on 32bit platforms ReadyCount int64 InFlightCount int64 MessageCount uint64 ZoneLocalMsgCount uint64 RegionLocalMsgCount uint64 GlobalMsgCount uint64 FinishCount uint64 RequeueCount uint64 pubCounts map[string]uint64 writeLock sync.RWMutex metaLock sync.RWMutex ID int64 nsqd *NSQD UserAgent string // original connection net.Conn // connections based on negotiated features tlsConn *tls.Conn flateWriter *flate.Writer // reading/writing interfaces Reader *bufio.Reader Writer *bufio.Writer OutputBufferSize int OutputBufferTimeout time.Duration HeartbeatInterval time.Duration MsgTimeout time.Duration State int32 ConnectTime time.Time Channel *Channel ReadyStateChan chan int ExitChan chan int ClientID string Hostname string TopologyRegion string TopologyZone string SampleRate int32 IdentifyEventChan chan identifyEvent SubEventChan chan *Channel TLS int32 Snappy int32 Deflate int32 // re-usable buffer for reading the 4-byte lengths off the wire lenBuf [4]byte lenSlice []byte AuthSecret string AuthState *auth.State } func newClientV2(id int64, conn net.Conn, nsqd *NSQD) *clientV2 { var identifier string if conn != nil { identifier, _, _ = net.SplitHostPort(conn.RemoteAddr().String()) } c := &clientV2{ ID: id, nsqd: nsqd, Conn: conn, Reader: bufio.NewReaderSize(conn, defaultBufferSize), Writer: bufio.NewWriterSize(conn, defaultBufferSize), OutputBufferSize: defaultBufferSize, OutputBufferTimeout: nsqd.getOpts().OutputBufferTimeout, MsgTimeout: nsqd.getOpts().MsgTimeout, // ReadyStateChan has a buffer of 1 to guarantee that in the event // there is a race the state update is not lost ReadyStateChan: make(chan int, 1), ExitChan: make(chan int), ConnectTime: time.Now(), State: stateInit, ClientID: identifier, Hostname: identifier, SubEventChan: make(chan *Channel, 1), IdentifyEventChan: make(chan identifyEvent, 1), // heartbeats are client configurable but default to 30s HeartbeatInterval: nsqd.getOpts().ClientTimeout / 2, pubCounts: make(map[string]uint64), } c.lenSlice = c.lenBuf[:] return c } func (c *clientV2) String() string { return c.RemoteAddr().String() } func (c *clientV2) Type() int { c.metaLock.RLock() hasPublished := len(c.pubCounts) > 0 c.metaLock.RUnlock() if hasPublished { return typeProducer } return typeConsumer } func (c *clientV2) Identify(data identifyDataV2) error { c.nsqd.logf(LOG_INFO, "[%s] IDENTIFY: %+v", c, data) c.metaLock.Lock() c.ClientID = data.ClientID c.Hostname = data.Hostname c.UserAgent = data.UserAgent c.TopologyRegion = data.TopologyRegion c.TopologyZone = data.TopologyZone c.metaLock.Unlock() err := c.SetHeartbeatInterval(data.HeartbeatInterval) if err != nil { return err } err = c.SetOutputBuffer(data.OutputBufferSize, data.OutputBufferTimeout) if err != nil { return err } err = c.SetSampleRate(data.SampleRate) if err != nil { return err } err = c.SetMsgTimeout(data.MsgTimeout) if err != nil { return err } ie := identifyEvent{ OutputBufferTimeout: c.OutputBufferTimeout, HeartbeatInterval: c.HeartbeatInterval, SampleRate: c.SampleRate, MsgTimeout: c.MsgTimeout, TopologyRegion: c.TopologyRegion, TopologyZone: c.TopologyZone, } // update the client's message pump select { case c.IdentifyEventChan <- ie: default: } return nil } func (c *clientV2) Stats(topicName string) ClientStats { c.metaLock.RLock() clientID := c.ClientID hostname := c.Hostname userAgent := c.UserAgent topologyZone := c.TopologyZone topologyRegion := c.TopologyRegion var identity string var identityURL string if c.AuthState != nil { identity = c.AuthState.Identity identityURL = c.AuthState.IdentityURL } pubCounts := make([]PubCount, 0, len(c.pubCounts)) for topic, count := range c.pubCounts { if len(topicName) > 0 && topic != topicName { continue } pubCounts = append(pubCounts, PubCount{ Topic: topic, Count: count, }) break } c.metaLock.RUnlock() stats := ClientV2Stats{ Version: "V2", RemoteAddress: c.RemoteAddr().String(), ClientID: clientID, Hostname: hostname, UserAgent: userAgent, State: atomic.LoadInt32(&c.State), ReadyCount: atomic.LoadInt64(&c.ReadyCount), InFlightCount: atomic.LoadInt64(&c.InFlightCount), MessageCount: atomic.LoadUint64(&c.MessageCount), ZoneLocalMsgCount: atomic.LoadUint64(&c.ZoneLocalMsgCount), RegionLocalMsgCount: atomic.LoadUint64(&c.RegionLocalMsgCount), GlobalMsgCount: atomic.LoadUint64(&c.GlobalMsgCount), FinishCount: atomic.LoadUint64(&c.FinishCount), RequeueCount: atomic.LoadUint64(&c.RequeueCount), ConnectTime: c.ConnectTime.Unix(), SampleRate: atomic.LoadInt32(&c.SampleRate), TLS: atomic.LoadInt32(&c.TLS) == 1, Deflate: atomic.LoadInt32(&c.Deflate) == 1, Snappy: atomic.LoadInt32(&c.Snappy) == 1, Authed: c.HasAuthorizations(), AuthIdentity: identity, AuthIdentityURL: identityURL, PubCounts: pubCounts, TopologyZone: topologyZone, TopologyRegion: topologyRegion, } if stats.TLS { p := prettyConnectionState{c.tlsConn.ConnectionState()} stats.CipherSuite = p.GetCipherSuite() stats.TLSVersion = p.GetVersion() stats.TLSNegotiatedProtocol = p.NegotiatedProtocol stats.TLSNegotiatedProtocolIsMutual = p.NegotiatedProtocolIsMutual } return stats } // struct to convert from integers to the human readable strings type prettyConnectionState struct { tls.ConnectionState } func (p *prettyConnectionState) GetCipherSuite() string { switch p.CipherSuite { case tls.TLS_RSA_WITH_RC4_128_SHA: return "TLS_RSA_WITH_RC4_128_SHA" case tls.TLS_RSA_WITH_3DES_EDE_CBC_SHA: return "TLS_RSA_WITH_3DES_EDE_CBC_SHA" case tls.TLS_RSA_WITH_AES_128_CBC_SHA: return "TLS_RSA_WITH_AES_128_CBC_SHA" case tls.TLS_RSA_WITH_AES_256_CBC_SHA: return "TLS_RSA_WITH_AES_256_CBC_SHA" case tls.TLS_ECDHE_ECDSA_WITH_RC4_128_SHA: return "TLS_ECDHE_ECDSA_WITH_RC4_128_SHA" case tls.TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA: return "TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA" case tls.TLS_ECDHE_ECDSA_WITH_AES_256_CBC_SHA: return "TLS_ECDHE_ECDSA_WITH_AES_256_CBC_SHA" case tls.TLS_ECDHE_RSA_WITH_RC4_128_SHA: return "TLS_ECDHE_RSA_WITH_RC4_128_SHA" case tls.TLS_ECDHE_RSA_WITH_3DES_EDE_CBC_SHA: return "TLS_ECDHE_RSA_WITH_3DES_EDE_CBC_SHA" case tls.TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA: return "TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA" case tls.TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA: return "TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA" case tls.TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256: return "TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256" case tls.TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256: return "TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256" } return fmt.Sprintf("Unknown %d", p.CipherSuite) } func (p *prettyConnectionState) GetVersion() string { switch p.Version { case tls.VersionTLS10: return "TLS1.0" case tls.VersionTLS11: return "TLS1.1" case tls.VersionTLS12: return "TLS1.2" case tls.VersionTLS13: return "TLS1.3" default: return fmt.Sprintf("Unknown %d", p.Version) } } func (c *clientV2) IsReadyForMessages() bool { if c.Channel.IsPaused() { return false } readyCount := atomic.LoadInt64(&c.ReadyCount) inFlightCount := atomic.LoadInt64(&c.InFlightCount) c.nsqd.logf(LOG_DEBUG, "[%s] state rdy: %4d inflt: %4d", c, readyCount, inFlightCount) if inFlightCount >= readyCount || readyCount <= 0 { return false } return true } func (c *clientV2) SetReadyCount(count int64) { oldCount := atomic.SwapInt64(&c.ReadyCount, count) if oldCount != count { c.tryUpdateReadyState() } } func (c *clientV2) tryUpdateReadyState() { // you can always *try* to write to ReadyStateChan because in the cases // where you cannot the message pump loop would have iterated anyway. // the atomic integer operations guarantee correctness of the value. select { case c.ReadyStateChan <- 1: default: } } func (c *clientV2) FinishedMessage() { atomic.AddUint64(&c.FinishCount, 1) atomic.AddInt64(&c.InFlightCount, -1) c.tryUpdateReadyState() } func (c *clientV2) Empty() { atomic.StoreInt64(&c.InFlightCount, 0) c.tryUpdateReadyState() } func (c *clientV2) SendingMessage() { atomic.AddInt64(&c.InFlightCount, 1) atomic.AddUint64(&c.MessageCount, 1) } func (c *clientV2) PublishedMessage(topic string, count uint64) { c.metaLock.Lock() c.pubCounts[topic] += count c.metaLock.Unlock() } func (c *clientV2) TimedOutMessage() { atomic.AddInt64(&c.InFlightCount, -1) c.tryUpdateReadyState() } func (c *clientV2) RequeuedMessage() { atomic.AddUint64(&c.RequeueCount, 1) atomic.AddInt64(&c.InFlightCount, -1) c.tryUpdateReadyState() } func (c *clientV2) StartClose() { // Force the client into ready 0 c.SetReadyCount(0) // mark this client as closing atomic.StoreInt32(&c.State, stateClosing) } func (c *clientV2) Pause() { c.tryUpdateReadyState() } func (c *clientV2) UnPause() { c.tryUpdateReadyState() } func (c *clientV2) SetHeartbeatInterval(desiredInterval int) error { c.writeLock.Lock() defer c.writeLock.Unlock() switch { case desiredInterval == -1: c.HeartbeatInterval = 0 case desiredInterval == 0: // do nothing (use default) case desiredInterval >= 1000 && desiredInterval <= int(c.nsqd.getOpts().MaxHeartbeatInterval/time.Millisecond): c.HeartbeatInterval = time.Duration(desiredInterval) * time.Millisecond default: return fmt.Errorf("heartbeat interval (%d) is invalid", desiredInterval) } return nil } func (c *clientV2) SetOutputBuffer(desiredSize int, desiredTimeout int) error { c.writeLock.Lock() defer c.writeLock.Unlock() switch { case desiredTimeout == -1: c.OutputBufferTimeout = 0 case desiredTimeout == 0: // do nothing (use default) case true && desiredTimeout >= int(c.nsqd.getOpts().MinOutputBufferTimeout/time.Millisecond) && desiredTimeout <= int(c.nsqd.getOpts().MaxOutputBufferTimeout/time.Millisecond): c.OutputBufferTimeout = time.Duration(desiredTimeout) * time.Millisecond default: return fmt.Errorf("output buffer timeout (%d) is invalid", desiredTimeout) } switch { case desiredSize == -1: // effectively no buffer (every write will go directly to the wrapped net.Conn) c.OutputBufferSize = 1 c.OutputBufferTimeout = 0 case desiredSize == 0: // do nothing (use default) case desiredSize >= 64 && desiredSize <= int(c.nsqd.getOpts().MaxOutputBufferSize): c.OutputBufferSize = desiredSize default: return fmt.Errorf("output buffer size (%d) is invalid", desiredSize) } if desiredSize != 0 { err := c.Writer.Flush() if err != nil { return err } c.Writer = bufio.NewWriterSize(c.Conn, c.OutputBufferSize) } return nil } func (c *clientV2) SetSampleRate(sampleRate int32) error { if sampleRate < 0 || sampleRate > 99 { return fmt.Errorf("sample rate (%d) is invalid", sampleRate) } atomic.StoreInt32(&c.SampleRate, sampleRate) return nil } func (c *clientV2) SetMsgTimeout(msgTimeout int) error { c.writeLock.Lock() defer c.writeLock.Unlock() switch { case msgTimeout == 0: // do nothing (use default) case msgTimeout >= 1000 && msgTimeout <= int(c.nsqd.getOpts().MaxMsgTimeout/time.Millisecond): c.MsgTimeout = time.Duration(msgTimeout) * time.Millisecond default: return fmt.Errorf("msg timeout (%d) is invalid", msgTimeout) } return nil } func (c *clientV2) UpgradeTLS() error { c.writeLock.Lock() defer c.writeLock.Unlock() tlsConn := tls.Server(c.Conn, c.nsqd.tlsConfig) tlsConn.SetDeadline(time.Now().Add(5 * time.Second)) err := tlsConn.Handshake() if err != nil { return err } c.tlsConn = tlsConn c.Reader = bufio.NewReaderSize(c.tlsConn, defaultBufferSize) c.Writer = bufio.NewWriterSize(c.tlsConn, c.OutputBufferSize) atomic.StoreInt32(&c.TLS, 1) return nil } func (c *clientV2) UpgradeDeflate(level int) error { c.writeLock.Lock() defer c.writeLock.Unlock() conn := c.Conn if c.tlsConn != nil { conn = c.tlsConn } c.Reader = bufio.NewReaderSize(flate.NewReader(conn), defaultBufferSize) fw, _ := flate.NewWriter(conn, level) c.flateWriter = fw c.Writer = bufio.NewWriterSize(fw, c.OutputBufferSize) atomic.StoreInt32(&c.Deflate, 1) return nil } func (c *clientV2) UpgradeSnappy() error { c.writeLock.Lock() defer c.writeLock.Unlock() conn := c.Conn if c.tlsConn != nil { conn = c.tlsConn } c.Reader = bufio.NewReaderSize(snappy.NewReader(conn), defaultBufferSize) //lint:ignore SA1019 NewWriter is deprecated by NewBufferedWriter, but we're doing our own buffering c.Writer = bufio.NewWriterSize(snappy.NewWriter(conn), c.OutputBufferSize) atomic.StoreInt32(&c.Snappy, 1) return nil } func (c *clientV2) Flush() error { var zeroTime time.Time if c.HeartbeatInterval > 0 { c.SetWriteDeadline(time.Now().Add(c.HeartbeatInterval)) } else { c.SetWriteDeadline(zeroTime) } err := c.Writer.Flush() if err != nil { return err } if c.flateWriter != nil { return c.flateWriter.Flush() } return nil } func (c *clientV2) QueryAuthd() error { remoteIP := "" if c.RemoteAddr().Network() == "tcp" { ip, _, err := net.SplitHostPort(c.String()) if err != nil { return err } remoteIP = ip } tlsEnabled := atomic.LoadInt32(&c.TLS) == 1 commonName := "" if tlsEnabled { tlsConnState := c.tlsConn.ConnectionState() if len(tlsConnState.PeerCertificates) > 0 { commonName = tlsConnState.PeerCertificates[0].Subject.CommonName } } authState, err := auth.QueryAnyAuthd(c.nsqd.getOpts().AuthHTTPAddresses, remoteIP, tlsEnabled, commonName, c.AuthSecret, c.nsqd.clientTLSConfig, c.nsqd.getOpts().HTTPClientConnectTimeout, c.nsqd.getOpts().HTTPClientRequestTimeout, c.nsqd.getOpts().AuthHTTPRequestMethod, ) if err != nil { return err } c.AuthState = authState return nil } func (c *clientV2) Auth(secret string) error { c.AuthSecret = secret return c.QueryAuthd() } func (c *clientV2) IsAuthorized(topic, channel string) (bool, error) { if c.AuthState == nil { return false, nil } if c.AuthState.IsExpired() { err := c.QueryAuthd() if err != nil { return false, err } } if c.AuthState.IsAllowed(topic, channel) { return true, nil } return false, nil } func (c *clientV2) HasAuthorizations() bool { if c.AuthState != nil { return len(c.AuthState.Authorizations) != 0 } return false } ================================================ FILE: nsqd/dqname.go ================================================ //go:build !windows // +build !windows package nsqd func getBackendName(topicName, channelName string) string { // backend names, for uniqueness, automatically include the topic... : backendName := topicName + ":" + channelName return backendName } ================================================ FILE: nsqd/dqname_windows.go ================================================ //go:build windows // +build windows package nsqd // On Windows, file names cannot contain colons. func getBackendName(topicName, channelName string) string { // backend names, for uniqueness, automatically include the topic... ; backendName := topicName + ";" + channelName return backendName } ================================================ FILE: nsqd/dummy_backend_queue.go ================================================ package nsqd type dummyBackendQueue struct { readChan chan []byte } func newDummyBackendQueue() BackendQueue { return &dummyBackendQueue{readChan: make(chan []byte)} } func (d *dummyBackendQueue) Put([]byte) error { return nil } func (d *dummyBackendQueue) ReadChan() <-chan []byte { return d.readChan } func (d *dummyBackendQueue) Close() error { return nil } func (d *dummyBackendQueue) Delete() error { return nil } func (d *dummyBackendQueue) Depth() int64 { return int64(0) } func (d *dummyBackendQueue) Empty() error { return nil } ================================================ FILE: nsqd/guid.go ================================================ package nsqd // the core algorithm here was borrowed from: // Blake Mizerany's `noeqd` https://github.com/bmizerany/noeqd // and indirectly: // Twitter's `snowflake` https://github.com/twitter/snowflake // only minor cleanup and changes to introduce a type, combine the concept // of workerID + datacenterId into a single identifier, and modify the // behavior when sequences rollover for our specific implementation needs import ( "encoding/hex" "errors" "sync" "time" ) const ( nodeIDBits = uint64(10) sequenceBits = uint64(12) nodeIDShift = sequenceBits timestampShift = sequenceBits + nodeIDBits sequenceMask = int64(-1) ^ (int64(-1) << sequenceBits) // ( 2012-10-28 16:23:42 UTC ).UnixNano() >> 20 twepoch = int64(1288834974288) ) var ErrTimeBackwards = errors.New("time has gone backwards") var ErrSequenceExpired = errors.New("sequence expired") var ErrIDBackwards = errors.New("ID went backward") type guid int64 type guidFactory struct { sync.Mutex nodeID int64 sequence int64 lastTimestamp int64 lastID guid } func NewGUIDFactory(nodeID int64) *guidFactory { return &guidFactory{ nodeID: nodeID, } } func (f *guidFactory) NewGUID() (guid, error) { f.Lock() // divide by 1048576, giving pseudo-milliseconds ts := time.Now().UnixNano() >> 20 if ts < f.lastTimestamp { f.Unlock() return 0, ErrTimeBackwards } if f.lastTimestamp == ts { f.sequence = (f.sequence + 1) & sequenceMask if f.sequence == 0 { f.Unlock() return 0, ErrSequenceExpired } } else { f.sequence = 0 } f.lastTimestamp = ts id := guid(((ts - twepoch) << timestampShift) | (f.nodeID << nodeIDShift) | f.sequence) if id <= f.lastID { f.Unlock() return 0, ErrIDBackwards } f.lastID = id f.Unlock() return id, nil } func (g guid) Hex() MessageID { var h MessageID var b [8]byte b[0] = byte(g >> 56) b[1] = byte(g >> 48) b[2] = byte(g >> 40) b[3] = byte(g >> 32) b[4] = byte(g >> 24) b[5] = byte(g >> 16) b[6] = byte(g >> 8) b[7] = byte(g) hex.Encode(h[:], b[:]) return h } ================================================ FILE: nsqd/guid_test.go ================================================ package nsqd import ( "testing" "unsafe" ) func BenchmarkGUIDCopy(b *testing.B) { source := make([]byte, 16) var dest MessageID for i := 0; i < b.N; i++ { copy(dest[:], source) } } func BenchmarkGUIDUnsafe(b *testing.B) { source := make([]byte, 16) var dest MessageID for i := 0; i < b.N; i++ { dest = *(*MessageID)(unsafe.Pointer(&source[0])) } _ = dest } func BenchmarkGUID(b *testing.B) { var okays, errors, fails int64 var previd guid factory := &guidFactory{} for i := 0; i < b.N; i++ { id, err := factory.NewGUID() if err != nil { errors++ } else if id == previd { fails++ b.Fail() } else { okays++ } id.Hex() } b.Logf("okays=%d errors=%d bads=%d", okays, errors, fails) } ================================================ FILE: nsqd/http.go ================================================ package nsqd import ( "bufio" "bytes" "encoding/json" "fmt" "io" "net" "net/http" "net/http/pprof" "net/url" "os" "reflect" "runtime" "runtime/debug" "strconv" "strings" "time" "github.com/julienschmidt/httprouter" "github.com/nsqio/nsq/internal/http_api" "github.com/nsqio/nsq/internal/lg" "github.com/nsqio/nsq/internal/protocol" "github.com/nsqio/nsq/internal/version" ) var boolParams = map[string]bool{ "true": true, "1": true, "false": false, "0": false, } type httpServer struct { nsqd *NSQD tlsEnabled bool tlsRequired bool router http.Handler } func newHTTPServer(nsqd *NSQD, tlsEnabled bool, tlsRequired bool) *httpServer { log := http_api.Log(nsqd.logf) router := httprouter.New() router.HandleMethodNotAllowed = true router.PanicHandler = http_api.LogPanicHandler(nsqd.logf) router.NotFound = http_api.LogNotFoundHandler(nsqd.logf) router.MethodNotAllowed = http_api.LogMethodNotAllowedHandler(nsqd.logf) s := &httpServer{ nsqd: nsqd, tlsEnabled: tlsEnabled, tlsRequired: tlsRequired, router: router, } router.Handle("GET", "/ping", http_api.Decorate(s.pingHandler, log, http_api.PlainText)) router.Handle("GET", "/info", http_api.Decorate(s.doInfo, log, http_api.V1)) // v1 negotiate router.Handle("POST", "/pub", http_api.Decorate(s.doPUB, http_api.V1)) router.Handle("POST", "/mpub", http_api.Decorate(s.doMPUB, http_api.V1)) router.Handle("GET", "/stats", http_api.Decorate(s.doStats, log, http_api.V1)) // only v1 router.Handle("POST", "/topic/create", http_api.Decorate(s.doCreateTopic, log, http_api.V1)) router.Handle("POST", "/topic/delete", http_api.Decorate(s.doDeleteTopic, log, http_api.V1)) router.Handle("POST", "/topic/empty", http_api.Decorate(s.doEmptyTopic, log, http_api.V1)) router.Handle("POST", "/topic/pause", http_api.Decorate(s.doPauseTopic, log, http_api.V1)) router.Handle("POST", "/topic/unpause", http_api.Decorate(s.doPauseTopic, log, http_api.V1)) router.Handle("POST", "/channel/create", http_api.Decorate(s.doCreateChannel, log, http_api.V1)) router.Handle("POST", "/channel/delete", http_api.Decorate(s.doDeleteChannel, log, http_api.V1)) router.Handle("POST", "/channel/empty", http_api.Decorate(s.doEmptyChannel, log, http_api.V1)) router.Handle("POST", "/channel/pause", http_api.Decorate(s.doPauseChannel, log, http_api.V1)) router.Handle("POST", "/channel/unpause", http_api.Decorate(s.doPauseChannel, log, http_api.V1)) router.Handle("GET", "/config/:opt", http_api.Decorate(s.doConfig, log, http_api.V1)) router.Handle("PUT", "/config/:opt", http_api.Decorate(s.doConfig, log, http_api.V1)) // debug router.HandlerFunc("GET", "/debug/pprof/", pprof.Index) router.HandlerFunc("GET", "/debug/pprof/cmdline", pprof.Cmdline) router.HandlerFunc("GET", "/debug/pprof/symbol", pprof.Symbol) router.HandlerFunc("POST", "/debug/pprof/symbol", pprof.Symbol) router.HandlerFunc("GET", "/debug/pprof/profile", pprof.Profile) router.Handler("GET", "/debug/pprof/heap", pprof.Handler("heap")) router.Handler("GET", "/debug/pprof/goroutine", pprof.Handler("goroutine")) router.Handler("GET", "/debug/pprof/block", pprof.Handler("block")) router.Handle("PUT", "/debug/setblockrate", http_api.Decorate(setBlockRateHandler, log, http_api.PlainText)) router.Handle("POST", "/debug/freememory", http_api.Decorate(freeMemory, log, http_api.PlainText)) router.Handler("GET", "/debug/pprof/threadcreate", pprof.Handler("threadcreate")) return s } func setBlockRateHandler(w http.ResponseWriter, req *http.Request, ps httprouter.Params) (interface{}, error) { rate, err := strconv.Atoi(req.FormValue("rate")) if err != nil { return nil, http_api.Err{http.StatusBadRequest, fmt.Sprintf("invalid block rate : %s", err.Error())} } runtime.SetBlockProfileRate(rate) return nil, nil } func freeMemory(w http.ResponseWriter, req *http.Request, ps httprouter.Params) (interface{}, error) { debug.FreeOSMemory() return nil, nil } func (s *httpServer) ServeHTTP(w http.ResponseWriter, req *http.Request) { if !s.tlsEnabled && s.tlsRequired { resp := fmt.Sprintf(`{"message": "TLS_REQUIRED", "https_port": %d}`, s.nsqd.RealHTTPSAddr().Port) w.Header().Set("X-NSQ-Content-Type", "nsq; version=1.0") w.Header().Set("Content-Type", "application/json; charset=utf-8") w.WriteHeader(403) io.WriteString(w, resp) return } s.router.ServeHTTP(w, req) } func (s *httpServer) pingHandler(w http.ResponseWriter, req *http.Request, ps httprouter.Params) (interface{}, error) { health := s.nsqd.GetHealth() if !s.nsqd.IsHealthy() { return nil, http_api.Err{500, health} } return health, nil } func (s *httpServer) doInfo(w http.ResponseWriter, req *http.Request, ps httprouter.Params) (interface{}, error) { hostname, err := os.Hostname() if err != nil { return nil, http_api.Err{500, err.Error()} } tcpPort := -1 // in case of unix socket if s.nsqd.RealTCPAddr().Network() == "tcp" { tcpPort = s.nsqd.RealTCPAddr().(*net.TCPAddr).Port } httpPort := -1 // in case of unix socket if s.nsqd.RealHTTPAddr().Network() == "tcp" { httpPort = s.nsqd.RealHTTPAddr().(*net.TCPAddr).Port } return struct { Version string `json:"version"` BroadcastAddress string `json:"broadcast_address"` Hostname string `json:"hostname"` HTTPPort int `json:"http_port"` TCPPort int `json:"tcp_port"` StartTime int64 `json:"start_time"` MaxHeartBeatInterval time.Duration `json:"max_heartbeat_interval"` MaxOutBufferSize int64 `json:"max_output_buffer_size"` MaxOutBufferTimeout time.Duration `json:"max_output_buffer_timeout"` MaxDeflateLevel int `json:"max_deflate_level"` TopologyZone string `json:"topology_zone"` TopologyRegion string `json:"topology_region"` }{ Version: version.Binary, BroadcastAddress: s.nsqd.getOpts().BroadcastAddress, Hostname: hostname, TCPPort: tcpPort, HTTPPort: httpPort, StartTime: s.nsqd.GetStartTime().Unix(), MaxHeartBeatInterval: s.nsqd.getOpts().MaxHeartbeatInterval, MaxOutBufferSize: s.nsqd.getOpts().MaxOutputBufferSize, MaxOutBufferTimeout: s.nsqd.getOpts().MaxOutputBufferTimeout, MaxDeflateLevel: s.nsqd.getOpts().MaxDeflateLevel, TopologyZone: s.nsqd.getOpts().TopologyZone, TopologyRegion: s.nsqd.getOpts().TopologyRegion, }, nil } func (s *httpServer) getExistingTopicFromQuery(req *http.Request) (*http_api.ReqParams, *Topic, string, error) { reqParams, err := http_api.NewReqParams(req) if err != nil { s.nsqd.logf(LOG_ERROR, "failed to parse request params - %s", err) return nil, nil, "", http_api.Err{400, "INVALID_REQUEST"} } topicName, channelName, err := http_api.GetTopicChannelArgs(reqParams) if err != nil { return nil, nil, "", http_api.Err{400, err.Error()} } topic, err := s.nsqd.GetExistingTopic(topicName) if err != nil { return nil, nil, "", http_api.Err{404, "TOPIC_NOT_FOUND"} } return reqParams, topic, channelName, err } func (s *httpServer) getTopicFromQuery(req *http.Request) (url.Values, *Topic, error) { reqParams, err := url.ParseQuery(req.URL.RawQuery) if err != nil { s.nsqd.logf(LOG_ERROR, "failed to parse request params - %s", err) return nil, nil, http_api.Err{400, "INVALID_REQUEST"} } topicNames, ok := reqParams["topic"] if !ok { return nil, nil, http_api.Err{400, "MISSING_ARG_TOPIC"} } topicName := topicNames[0] if !protocol.IsValidTopicName(topicName) { return nil, nil, http_api.Err{400, "INVALID_TOPIC"} } return reqParams, s.nsqd.GetTopic(topicName), nil } func (s *httpServer) doPUB(w http.ResponseWriter, req *http.Request, ps httprouter.Params) (interface{}, error) { // TODO: one day I'd really like to just error on chunked requests // to be able to fail "too big" requests before we even read if req.ContentLength > s.nsqd.getOpts().MaxMsgSize { return nil, http_api.Err{413, "MSG_TOO_BIG"} } // add 1 so that it's greater than our max when we test for it // (LimitReader returns a "fake" EOF) readMax := s.nsqd.getOpts().MaxMsgSize + 1 body, err := io.ReadAll(io.LimitReader(req.Body, readMax)) if err != nil { return nil, http_api.Err{500, "INTERNAL_ERROR"} } if int64(len(body)) == readMax { return nil, http_api.Err{413, "MSG_TOO_BIG"} } if len(body) == 0 { return nil, http_api.Err{400, "MSG_EMPTY"} } reqParams, topic, err := s.getTopicFromQuery(req) if err != nil { return nil, err } var deferred time.Duration if ds, ok := reqParams["defer"]; ok { var di int64 di, err = strconv.ParseInt(ds[0], 10, 64) if err != nil { return nil, http_api.Err{400, "INVALID_DEFER"} } deferred = time.Duration(di) * time.Millisecond if deferred < 0 || deferred > s.nsqd.getOpts().MaxDeferTimeout { return nil, http_api.Err{400, "INVALID_DEFER"} } } msg := NewMessage(topic.GenerateID(), body) msg.deferred = deferred err = topic.PutMessage(msg) if err != nil { return nil, http_api.Err{503, "EXITING"} } return "OK", nil } func (s *httpServer) doMPUB(w http.ResponseWriter, req *http.Request, ps httprouter.Params) (interface{}, error) { var msgs []*Message var exit bool // TODO: one day I'd really like to just error on chunked requests // to be able to fail "too big" requests before we even read if req.ContentLength > s.nsqd.getOpts().MaxBodySize { return nil, http_api.Err{413, "BODY_TOO_BIG"} } reqParams, topic, err := s.getTopicFromQuery(req) if err != nil { return nil, err } // text mode is default, but unrecognized binary opt considered true binaryMode := false if vals, ok := reqParams["binary"]; ok { if binaryMode, ok = boolParams[vals[0]]; !ok { binaryMode = true s.nsqd.logf(LOG_WARN, "deprecated value '%s' used for /mpub binary param", vals[0]) } } if binaryMode { tmp := make([]byte, 4) msgs, err = readMPUB(req.Body, tmp, topic, s.nsqd.getOpts().MaxMsgSize, s.nsqd.getOpts().MaxBodySize) if err != nil { return nil, http_api.Err{413, err.(*protocol.FatalClientErr).Code[2:]} } } else { // add 1 so that it's greater than our max when we test for it // (LimitReader returns a "fake" EOF) readMax := s.nsqd.getOpts().MaxBodySize + 1 rdr := bufio.NewReader(io.LimitReader(req.Body, readMax)) total := 0 for !exit { var block []byte block, err = rdr.ReadBytes('\n') if err != nil { if err != io.EOF { return nil, http_api.Err{500, "INTERNAL_ERROR"} } exit = true } total += len(block) if int64(total) == readMax { return nil, http_api.Err{413, "BODY_TOO_BIG"} } if len(block) > 0 && block[len(block)-1] == '\n' { block = block[:len(block)-1] } // silently discard 0 length messages // this maintains the behavior pre 0.2.22 if len(block) == 0 { continue } if int64(len(block)) > s.nsqd.getOpts().MaxMsgSize { return nil, http_api.Err{413, "MSG_TOO_BIG"} } msg := NewMessage(topic.GenerateID(), block) msgs = append(msgs, msg) } } err = topic.PutMessages(msgs) if err != nil { return nil, http_api.Err{503, "EXITING"} } return "OK", nil } func (s *httpServer) doCreateTopic(w http.ResponseWriter, req *http.Request, ps httprouter.Params) (interface{}, error) { _, _, err := s.getTopicFromQuery(req) return nil, err } func (s *httpServer) doEmptyTopic(w http.ResponseWriter, req *http.Request, ps httprouter.Params) (interface{}, error) { reqParams, err := http_api.NewReqParams(req) if err != nil { s.nsqd.logf(LOG_ERROR, "failed to parse request params - %s", err) return nil, http_api.Err{400, "INVALID_REQUEST"} } topicName, err := reqParams.Get("topic") if err != nil { return nil, http_api.Err{400, "MISSING_ARG_TOPIC"} } if !protocol.IsValidTopicName(topicName) { return nil, http_api.Err{400, "INVALID_TOPIC"} } topic, err := s.nsqd.GetExistingTopic(topicName) if err != nil { return nil, http_api.Err{404, "TOPIC_NOT_FOUND"} } err = topic.Empty() if err != nil { return nil, http_api.Err{500, "INTERNAL_ERROR"} } return nil, nil } func (s *httpServer) doDeleteTopic(w http.ResponseWriter, req *http.Request, ps httprouter.Params) (interface{}, error) { reqParams, err := http_api.NewReqParams(req) if err != nil { s.nsqd.logf(LOG_ERROR, "failed to parse request params - %s", err) return nil, http_api.Err{400, "INVALID_REQUEST"} } topicName, err := reqParams.Get("topic") if err != nil { return nil, http_api.Err{400, "MISSING_ARG_TOPIC"} } err = s.nsqd.DeleteExistingTopic(topicName) if err != nil { return nil, http_api.Err{404, "TOPIC_NOT_FOUND"} } return nil, nil } func (s *httpServer) doPauseTopic(w http.ResponseWriter, req *http.Request, ps httprouter.Params) (interface{}, error) { reqParams, err := http_api.NewReqParams(req) if err != nil { s.nsqd.logf(LOG_ERROR, "failed to parse request params - %s", err) return nil, http_api.Err{400, "INVALID_REQUEST"} } topicName, err := reqParams.Get("topic") if err != nil { return nil, http_api.Err{400, "MISSING_ARG_TOPIC"} } topic, err := s.nsqd.GetExistingTopic(topicName) if err != nil { return nil, http_api.Err{404, "TOPIC_NOT_FOUND"} } if strings.Contains(req.URL.Path, "unpause") { err = topic.UnPause() } else { err = topic.Pause() } if err != nil { s.nsqd.logf(LOG_ERROR, "failure in %s - %s", req.URL.Path, err) return nil, http_api.Err{500, "INTERNAL_ERROR"} } // pro-actively persist metadata so in case of process failure // nsqd won't suddenly (un)pause a topic s.nsqd.Lock() s.nsqd.PersistMetadata() s.nsqd.Unlock() return nil, nil } func (s *httpServer) doCreateChannel(w http.ResponseWriter, req *http.Request, ps httprouter.Params) (interface{}, error) { _, topic, channelName, err := s.getExistingTopicFromQuery(req) if err != nil { return nil, err } topic.GetChannel(channelName) return nil, nil } func (s *httpServer) doEmptyChannel(w http.ResponseWriter, req *http.Request, ps httprouter.Params) (interface{}, error) { _, topic, channelName, err := s.getExistingTopicFromQuery(req) if err != nil { return nil, err } channel, err := topic.GetExistingChannel(channelName) if err != nil { return nil, http_api.Err{404, "CHANNEL_NOT_FOUND"} } err = channel.Empty() if err != nil { return nil, http_api.Err{500, "INTERNAL_ERROR"} } return nil, nil } func (s *httpServer) doDeleteChannel(w http.ResponseWriter, req *http.Request, ps httprouter.Params) (interface{}, error) { _, topic, channelName, err := s.getExistingTopicFromQuery(req) if err != nil { return nil, err } err = topic.DeleteExistingChannel(channelName) if err != nil { return nil, http_api.Err{404, "CHANNEL_NOT_FOUND"} } return nil, nil } func (s *httpServer) doPauseChannel(w http.ResponseWriter, req *http.Request, ps httprouter.Params) (interface{}, error) { _, topic, channelName, err := s.getExistingTopicFromQuery(req) if err != nil { return nil, err } channel, err := topic.GetExistingChannel(channelName) if err != nil { return nil, http_api.Err{404, "CHANNEL_NOT_FOUND"} } if strings.Contains(req.URL.Path, "unpause") { err = channel.UnPause() } else { err = channel.Pause() } if err != nil { s.nsqd.logf(LOG_ERROR, "failure in %s - %s", req.URL.Path, err) return nil, http_api.Err{500, "INTERNAL_ERROR"} } // pro-actively persist metadata so in case of process failure // nsqd won't suddenly (un)pause a channel s.nsqd.Lock() s.nsqd.PersistMetadata() s.nsqd.Unlock() return nil, nil } func (s *httpServer) doStats(w http.ResponseWriter, req *http.Request, ps httprouter.Params) (interface{}, error) { reqParams, err := http_api.NewReqParams(req) if err != nil { s.nsqd.logf(LOG_ERROR, "failed to parse request params - %s", err) return nil, http_api.Err{400, "INVALID_REQUEST"} } formatString, _ := reqParams.Get("format") topicName, _ := reqParams.Get("topic") channelName, _ := reqParams.Get("channel") includeClientsParam, _ := reqParams.Get("include_clients") includeMemParam, _ := reqParams.Get("include_mem") jsonFormat := formatString == "json" includeClients, ok := boolParams[includeClientsParam] if !ok { includeClients = true } includeMem, ok := boolParams[includeMemParam] if !ok { includeMem = true } stats := s.nsqd.GetStats(topicName, channelName, includeClients) health := s.nsqd.GetHealth() startTime := s.nsqd.GetStartTime() uptime := time.Since(startTime) var ms *memStats if includeMem { m := getMemStats() ms = &m } if !jsonFormat { return s.printStats(stats, ms, health, startTime, uptime), nil } // TODO: should producer stats be hung off topics? return struct { Version string `json:"version"` Health string `json:"health"` StartTime int64 `json:"start_time"` Topics []TopicStats `json:"topics"` Memory *memStats `json:"memory,omitempty"` Producers []ClientStats `json:"producers"` }{version.Binary, health, startTime.Unix(), stats.Topics, ms, stats.Producers}, nil } func (s *httpServer) printStats(stats Stats, ms *memStats, health string, startTime time.Time, uptime time.Duration) []byte { var buf bytes.Buffer w := &buf fmt.Fprintf(w, "%s\n", version.String("nsqd")) fmt.Fprintf(w, "start_time %v\n", startTime.Format(time.RFC3339)) fmt.Fprintf(w, "uptime %s\n", uptime) fmt.Fprintf(w, "\nHealth: %s\n", health) if ms != nil { fmt.Fprintf(w, "\nMemory:\n") fmt.Fprintf(w, " %-25s\t%d\n", "heap_objects", ms.HeapObjects) fmt.Fprintf(w, " %-25s\t%d\n", "heap_idle_bytes", ms.HeapIdleBytes) fmt.Fprintf(w, " %-25s\t%d\n", "heap_in_use_bytes", ms.HeapInUseBytes) fmt.Fprintf(w, " %-25s\t%d\n", "heap_released_bytes", ms.HeapReleasedBytes) fmt.Fprintf(w, " %-25s\t%d\n", "gc_pause_usec_100", ms.GCPauseUsec100) fmt.Fprintf(w, " %-25s\t%d\n", "gc_pause_usec_99", ms.GCPauseUsec99) fmt.Fprintf(w, " %-25s\t%d\n", "gc_pause_usec_95", ms.GCPauseUsec95) fmt.Fprintf(w, " %-25s\t%d\n", "next_gc_bytes", ms.NextGCBytes) fmt.Fprintf(w, " %-25s\t%d\n", "gc_total_runs", ms.GCTotalRuns) } if len(stats.Topics) == 0 { fmt.Fprintf(w, "\nTopics: None\n") } else { fmt.Fprintf(w, "\nTopics:") } for _, t := range stats.Topics { var pausedPrefix string if t.Paused { pausedPrefix = "*P " } else { pausedPrefix = " " } fmt.Fprintf(w, "\n%s[%-15s] depth: %-5d be-depth: %-5d msgs: %-8d e2e%%: %s\n", pausedPrefix, t.TopicName, t.Depth, t.BackendDepth, t.MessageCount, t.E2eProcessingLatency, ) for _, c := range t.Channels { if c.Paused { pausedPrefix = " *P " } else { pausedPrefix = " " } fmt.Fprintf(w, "%s[%-25s] depth: %-5d be-depth: %-5d inflt: %-4d def: %-4d re-q: %-5d timeout: %-5d msgs: %-8d e2e%%: %s\n", pausedPrefix, c.ChannelName, c.Depth, c.BackendDepth, c.InFlightCount, c.DeferredCount, c.RequeueCount, c.TimeoutCount, c.MessageCount, c.E2eProcessingLatency, ) for _, client := range c.Clients { fmt.Fprintf(w, " %s\n", client) } } } if len(stats.Producers) == 0 { fmt.Fprintf(w, "\nProducers: None\n") } else { fmt.Fprintf(w, "\nProducers:\n") for _, client := range stats.Producers { fmt.Fprintf(w, " %s\n", client) } } return buf.Bytes() } func (s *httpServer) doConfig(w http.ResponseWriter, req *http.Request, ps httprouter.Params) (interface{}, error) { opt := ps.ByName("opt") if req.Method == "PUT" { // add 1 so that it's greater than our max when we test for it // (LimitReader returns a "fake" EOF) readMax := s.nsqd.getOpts().MaxMsgSize + 1 body, err := io.ReadAll(io.LimitReader(req.Body, readMax)) if err != nil { return nil, http_api.Err{500, "INTERNAL_ERROR"} } if int64(len(body)) == readMax || len(body) == 0 { return nil, http_api.Err{413, "INVALID_VALUE"} } opts := *s.nsqd.getOpts() switch opt { case "nsqlookupd_tcp_addresses": err := json.Unmarshal(body, &opts.NSQLookupdTCPAddresses) if err != nil { return nil, http_api.Err{400, "INVALID_VALUE"} } case "log_level": logLevelStr := string(body) logLevel, err := lg.ParseLogLevel(logLevelStr) if err != nil { return nil, http_api.Err{400, "INVALID_VALUE"} } opts.LogLevel = logLevel default: return nil, http_api.Err{400, "INVALID_OPTION"} } s.nsqd.swapOpts(&opts) s.nsqd.triggerOptsNotification() } v, ok := getOptByCfgName(s.nsqd.getOpts(), opt) if !ok { return nil, http_api.Err{400, "INVALID_OPTION"} } return v, nil } func getOptByCfgName(opts interface{}, name string) (interface{}, bool) { val := reflect.ValueOf(opts).Elem() typ := val.Type() for i := 0; i < typ.NumField(); i++ { field := typ.Field(i) flagName := field.Tag.Get("flag") cfgName := field.Tag.Get("cfg") if flagName == "" { continue } if cfgName == "" { cfgName = strings.Replace(flagName, "-", "_", -1) } if name != cfgName { continue } return val.FieldByName(field.Name).Interface(), true } return nil, false } ================================================ FILE: nsqd/http_test.go ================================================ package nsqd import ( "bytes" "crypto/tls" "encoding/json" "fmt" "io" "net" "net/http" "os" "runtime" "strconv" "strings" "sync" "testing" "time" "github.com/nsqio/go-nsq" "github.com/nsqio/nsq/internal/http_api" "github.com/nsqio/nsq/internal/test" "github.com/nsqio/nsq/internal/version" "github.com/nsqio/nsq/nsqlookupd" ) type ErrMessage struct { Message string `json:"message"` } type InfoDoc struct { Version string `json:"version"` BroadcastAddress string `json:"broadcast_address"` Hostname string `json:"hostname"` HTTPPort int `json:"http_port"` TCPPort int `json:"tcp_port"` StartTime int64 `json:"start_time"` } func TestHTTPpub(t *testing.T) { opts := NewOptions() opts.Logger = test.NewTestLogger(t) _, httpAddr, nsqd := mustStartNSQD(opts) defer os.RemoveAll(opts.DataPath) defer nsqd.Exit() topicName := "test_http_pub" + strconv.Itoa(int(time.Now().Unix())) topic := nsqd.GetTopic(topicName) buf := bytes.NewBuffer([]byte("test message")) url := fmt.Sprintf("http://%s/pub?topic=%s", httpAddr, topicName) resp, err := http.Post(url, "application/octet-stream", buf) test.Nil(t, err) defer resp.Body.Close() body, _ := io.ReadAll(resp.Body) test.Equal(t, "OK", string(body)) time.Sleep(5 * time.Millisecond) test.Equal(t, int64(1), topic.Depth()) } func TestHTTPpubEmpty(t *testing.T) { opts := NewOptions() opts.Logger = test.NewTestLogger(t) _, httpAddr, nsqd := mustStartNSQD(opts) defer os.RemoveAll(opts.DataPath) defer nsqd.Exit() topicName := "test_http_pub_empty" + strconv.Itoa(int(time.Now().Unix())) topic := nsqd.GetTopic(topicName) buf := bytes.NewBuffer([]byte("")) url := fmt.Sprintf("http://%s/pub?topic=%s", httpAddr, topicName) resp, err := http.Post(url, "application/octet-stream", buf) test.Nil(t, err) defer resp.Body.Close() body, _ := io.ReadAll(resp.Body) test.Equal(t, 400, resp.StatusCode) test.Equal(t, `{"message":"MSG_EMPTY"}`, string(body)) time.Sleep(5 * time.Millisecond) test.Equal(t, int64(0), topic.Depth()) } func TestHTTPmpub(t *testing.T) { opts := NewOptions() opts.Logger = test.NewTestLogger(t) _, httpAddr, nsqd := mustStartNSQD(opts) defer os.RemoveAll(opts.DataPath) defer nsqd.Exit() topicName := "test_http_mpub" + strconv.Itoa(int(time.Now().Unix())) topic := nsqd.GetTopic(topicName) msg := []byte("test message") msgs := make([][]byte, 4) for i := range msgs { msgs[i] = msg } buf := bytes.NewBuffer(bytes.Join(msgs, []byte("\n"))) url := fmt.Sprintf("http://%s/mpub?topic=%s", httpAddr, topicName) resp, err := http.Post(url, "application/octet-stream", buf) test.Nil(t, err) defer resp.Body.Close() body, _ := io.ReadAll(resp.Body) test.Equal(t, "OK", string(body)) time.Sleep(5 * time.Millisecond) test.Equal(t, int64(4), topic.Depth()) } func TestHTTPmpubEmpty(t *testing.T) { opts := NewOptions() opts.Logger = test.NewTestLogger(t) _, httpAddr, nsqd := mustStartNSQD(opts) defer os.RemoveAll(opts.DataPath) defer nsqd.Exit() topicName := "test_http_mpub_empty" + strconv.Itoa(int(time.Now().Unix())) topic := nsqd.GetTopic(topicName) msg := []byte("test message") msgs := make([][]byte, 4) for i := range msgs { msgs[i] = msg } buf := bytes.NewBuffer(bytes.Join(msgs, []byte("\n"))) _, err := buf.Write([]byte("\n")) test.Nil(t, err) url := fmt.Sprintf("http://%s/mpub?topic=%s", httpAddr, topicName) resp, err := http.Post(url, "application/octet-stream", buf) test.Nil(t, err) defer resp.Body.Close() body, _ := io.ReadAll(resp.Body) test.Equal(t, "OK", string(body)) time.Sleep(5 * time.Millisecond) test.Equal(t, int64(4), topic.Depth()) } func TestHTTPmpubBinary(t *testing.T) { opts := NewOptions() opts.Logger = test.NewTestLogger(t) _, httpAddr, nsqd := mustStartNSQD(opts) defer os.RemoveAll(opts.DataPath) defer nsqd.Exit() topicName := "test_http_mpub_bin" + strconv.Itoa(int(time.Now().Unix())) topic := nsqd.GetTopic(topicName) mpub := make([][]byte, 5) for i := range mpub { mpub[i] = make([]byte, 100) } cmd, _ := nsq.MultiPublish(topicName, mpub) buf := bytes.NewBuffer(cmd.Body) url := fmt.Sprintf("http://%s/mpub?topic=%s&binary=true", httpAddr, topicName) resp, err := http.Post(url, "application/octet-stream", buf) test.Nil(t, err) defer resp.Body.Close() body, _ := io.ReadAll(resp.Body) test.Equal(t, "OK", string(body)) time.Sleep(5 * time.Millisecond) test.Equal(t, int64(5), topic.Depth()) } func TestHTTPmpubForNonNormalizedBinaryParam(t *testing.T) { opts := NewOptions() opts.Logger = test.NewTestLogger(t) _, httpAddr, nsqd := mustStartNSQD(opts) defer os.RemoveAll(opts.DataPath) defer nsqd.Exit() topicName := "test_http_mpub_bin" + strconv.Itoa(int(time.Now().Unix())) topic := nsqd.GetTopic(topicName) mpub := make([][]byte, 5) for i := range mpub { mpub[i] = make([]byte, 100) } cmd, _ := nsq.MultiPublish(topicName, mpub) buf := bytes.NewBuffer(cmd.Body) url := fmt.Sprintf("http://%s/mpub?topic=%s&binary=non_normalized_binary_param", httpAddr, topicName) resp, err := http.Post(url, "application/octet-stream", buf) test.Nil(t, err) defer resp.Body.Close() body, _ := io.ReadAll(resp.Body) test.Equal(t, "OK", string(body)) time.Sleep(5 * time.Millisecond) test.Equal(t, int64(5), topic.Depth()) } func TestHTTPpubDefer(t *testing.T) { opts := NewOptions() opts.Logger = test.NewTestLogger(t) _, httpAddr, nsqd := mustStartNSQD(opts) defer os.RemoveAll(opts.DataPath) defer nsqd.Exit() topicName := "test_http_pub_defer" + strconv.Itoa(int(time.Now().Unix())) topic := nsqd.GetTopic(topicName) ch := topic.GetChannel("ch") buf := bytes.NewBuffer([]byte("test message")) url := fmt.Sprintf("http://%s/pub?topic=%s&defer=%d", httpAddr, topicName, 1000) resp, err := http.Post(url, "application/octet-stream", buf) test.Nil(t, err) defer resp.Body.Close() body, _ := io.ReadAll(resp.Body) test.Equal(t, "OK", string(body)) time.Sleep(5 * time.Millisecond) ch.deferredMutex.Lock() numDef := len(ch.deferredMessages) ch.deferredMutex.Unlock() test.Equal(t, 1, numDef) } func TestHTTPSRequire(t *testing.T) { opts := NewOptions() opts.Logger = test.NewTestLogger(t) opts.LogLevel = LOG_DEBUG opts.TLSCert = "./test/certs/server.pem" opts.TLSKey = "./test/certs/server.key" opts.TLSClientAuthPolicy = "require" _, httpAddr, nsqd := mustStartNSQD(opts) defer os.RemoveAll(opts.DataPath) defer nsqd.Exit() topicName := "test_http_pub_req" + strconv.Itoa(int(time.Now().Unix())) topic := nsqd.GetTopic(topicName) buf := bytes.NewBuffer([]byte("test message")) url := fmt.Sprintf("http://%s/pub?topic=%s", httpAddr, topicName) resp, _ := http.Post(url, "application/octet-stream", buf) test.Equal(t, 403, resp.StatusCode) httpsAddr := nsqd.httpsListener.Addr().(*net.TCPAddr) cert, err := tls.LoadX509KeyPair("./test/certs/cert.pem", "./test/certs/key.pem") test.Nil(t, err) tlsConfig := &tls.Config{ Certificates: []tls.Certificate{cert}, InsecureSkipVerify: true, MinVersion: 0, } transport := &http.Transport{ TLSClientConfig: tlsConfig, } client := &http.Client{Transport: transport} buf = bytes.NewBuffer([]byte("test message")) url = fmt.Sprintf("https://%s/pub?topic=%s", httpsAddr, topicName) resp, err = client.Post(url, "application/octet-stream", buf) test.Nil(t, err) defer resp.Body.Close() body, _ := io.ReadAll(resp.Body) test.Equal(t, "OK", string(body)) time.Sleep(5 * time.Millisecond) test.Equal(t, int64(1), topic.Depth()) } func TestHTTPSRequireVerify(t *testing.T) { opts := NewOptions() opts.Logger = test.NewTestLogger(t) opts.LogLevel = LOG_DEBUG opts.TLSCert = "./test/certs/server.pem" opts.TLSKey = "./test/certs/server.key" opts.TLSRootCAFile = "./test/certs/ca.pem" opts.TLSClientAuthPolicy = "require-verify" _, httpAddr, nsqd := mustStartNSQD(opts) defer os.RemoveAll(opts.DataPath) defer nsqd.Exit() httpsAddr := nsqd.httpsListener.Addr().(*net.TCPAddr) topicName := "test_http_pub_req_verf" + strconv.Itoa(int(time.Now().Unix())) topic := nsqd.GetTopic(topicName) // no cert buf := bytes.NewBuffer([]byte("test message")) url := fmt.Sprintf("http://%s/pub?topic=%s", httpAddr, topicName) resp, _ := http.Post(url, "application/octet-stream", buf) test.Equal(t, 403, resp.StatusCode) // unsigned cert cert, err := tls.LoadX509KeyPair("./test/certs/cert.pem", "./test/certs/key.pem") test.Nil(t, err) tlsConfig := &tls.Config{ Certificates: []tls.Certificate{cert}, InsecureSkipVerify: true, } transport := &http.Transport{ TLSClientConfig: tlsConfig, } client := &http.Client{Transport: transport} buf = bytes.NewBuffer([]byte("test message")) url = fmt.Sprintf("https://%s/pub?topic=%s", httpsAddr, topicName) _, err = client.Post(url, "application/octet-stream", buf) test.NotNil(t, err) // signed cert cert, err = tls.LoadX509KeyPair("./test/certs/client.pem", "./test/certs/client.key") test.Nil(t, err) tlsConfig = &tls.Config{ Certificates: []tls.Certificate{cert}, InsecureSkipVerify: true, } transport = &http.Transport{ TLSClientConfig: tlsConfig, } client = &http.Client{Transport: transport} buf = bytes.NewBuffer([]byte("test message")) url = fmt.Sprintf("https://%s/pub?topic=%s", httpsAddr, topicName) resp, err = client.Post(url, "application/octet-stream", buf) test.Nil(t, err) defer resp.Body.Close() body, _ := io.ReadAll(resp.Body) test.Equal(t, "OK", string(body)) time.Sleep(5 * time.Millisecond) test.Equal(t, int64(1), topic.Depth()) } func TestTLSRequireVerifyExceptHTTP(t *testing.T) { opts := NewOptions() opts.Logger = test.NewTestLogger(t) opts.LogLevel = LOG_DEBUG opts.TLSCert = "./test/certs/server.pem" opts.TLSKey = "./test/certs/server.key" opts.TLSRootCAFile = "./test/certs/ca.pem" opts.TLSClientAuthPolicy = "require-verify" opts.TLSRequired = TLSRequiredExceptHTTP _, httpAddr, nsqd := mustStartNSQD(opts) defer os.RemoveAll(opts.DataPath) defer nsqd.Exit() topicName := "test_http_req_verf_except_http" + strconv.Itoa(int(time.Now().Unix())) topic := nsqd.GetTopic(topicName) // no cert buf := bytes.NewBuffer([]byte("test message")) url := fmt.Sprintf("http://%s/pub?topic=%s", httpAddr, topicName) resp, err := http.Post(url, "application/octet-stream", buf) test.Nil(t, err) defer resp.Body.Close() body, _ := io.ReadAll(resp.Body) test.Equal(t, "OK", string(body)) time.Sleep(5 * time.Millisecond) test.Equal(t, int64(1), topic.Depth()) } func TestHTTPV1TopicChannel(t *testing.T) { opts := NewOptions() opts.Logger = test.NewTestLogger(t) _, httpAddr, nsqd := mustStartNSQD(opts) defer os.RemoveAll(opts.DataPath) defer nsqd.Exit() topicName := "test_http_topic_channel2" + strconv.Itoa(int(time.Now().Unix())) channelName := "ch2" url := fmt.Sprintf("http://%s/topic/create?topic=%s", httpAddr, topicName) resp, err := http.Post(url, "application/json", nil) test.Nil(t, err) test.Equal(t, 200, resp.StatusCode) body, _ := io.ReadAll(resp.Body) resp.Body.Close() test.Equal(t, "", string(body)) test.Equal(t, "nsq; version=1.0", resp.Header.Get("X-NSQ-Content-Type")) url = fmt.Sprintf("http://%s/channel/create?topic=%s&channel=%s", httpAddr, topicName, channelName) resp, err = http.Post(url, "application/json", nil) test.Nil(t, err) test.Equal(t, 200, resp.StatusCode) body, _ = io.ReadAll(resp.Body) resp.Body.Close() test.Equal(t, "", string(body)) test.Equal(t, "nsq; version=1.0", resp.Header.Get("X-NSQ-Content-Type")) topic, err := nsqd.GetExistingTopic(topicName) test.Nil(t, err) test.NotNil(t, topic) channel, err := topic.GetExistingChannel(channelName) test.Nil(t, err) test.NotNil(t, channel) em := ErrMessage{} url = fmt.Sprintf("http://%s/topic/pause", httpAddr) resp, err = http.Post(url, "application/json", nil) test.Nil(t, err) test.Equal(t, 400, resp.StatusCode) test.Equal(t, "Bad Request", http.StatusText(resp.StatusCode)) body, _ = io.ReadAll(resp.Body) resp.Body.Close() t.Logf("%s", body) err = json.Unmarshal(body, &em) test.Nil(t, err) test.Equal(t, "MISSING_ARG_TOPIC", em.Message) url = fmt.Sprintf("http://%s/topic/pause?topic=%s", httpAddr, topicName+"abc") resp, err = http.Post(url, "application/json", nil) test.Nil(t, err) test.Equal(t, 404, resp.StatusCode) test.Equal(t, "Not Found", http.StatusText(resp.StatusCode)) body, _ = io.ReadAll(resp.Body) resp.Body.Close() t.Logf("%s", body) err = json.Unmarshal(body, &em) test.Nil(t, err) test.Equal(t, "TOPIC_NOT_FOUND", em.Message) url = fmt.Sprintf("http://%s/topic/pause?topic=%s", httpAddr, topicName) resp, err = http.Post(url, "application/json", nil) test.Nil(t, err) test.Equal(t, 200, resp.StatusCode) body, _ = io.ReadAll(resp.Body) resp.Body.Close() test.Equal(t, "", string(body)) test.Equal(t, "nsq; version=1.0", resp.Header.Get("X-NSQ-Content-Type")) test.Equal(t, true, topic.IsPaused()) url = fmt.Sprintf("http://%s/topic/unpause?topic=%s", httpAddr, topicName) resp, err = http.Post(url, "application/json", nil) test.Nil(t, err) test.Equal(t, 200, resp.StatusCode) body, _ = io.ReadAll(resp.Body) resp.Body.Close() test.Equal(t, "", string(body)) test.Equal(t, "nsq; version=1.0", resp.Header.Get("X-NSQ-Content-Type")) test.Equal(t, false, topic.IsPaused()) url = fmt.Sprintf("http://%s/channel/pause?topic=%s&channel=%s", httpAddr, topicName, channelName) resp, err = http.Post(url, "application/json", nil) test.Nil(t, err) test.Equal(t, 200, resp.StatusCode) body, _ = io.ReadAll(resp.Body) resp.Body.Close() test.Equal(t, "", string(body)) test.Equal(t, "nsq; version=1.0", resp.Header.Get("X-NSQ-Content-Type")) test.Equal(t, true, channel.IsPaused()) url = fmt.Sprintf("http://%s/channel/unpause?topic=%s&channel=%s", httpAddr, topicName, channelName) resp, err = http.Post(url, "application/json", nil) test.Nil(t, err) test.Equal(t, 200, resp.StatusCode) body, _ = io.ReadAll(resp.Body) resp.Body.Close() test.Equal(t, "", string(body)) test.Equal(t, "nsq; version=1.0", resp.Header.Get("X-NSQ-Content-Type")) test.Equal(t, false, channel.IsPaused()) url = fmt.Sprintf("http://%s/channel/delete?topic=%s&channel=%s", httpAddr, topicName, channelName) resp, err = http.Post(url, "application/json", nil) test.Nil(t, err) test.Equal(t, 200, resp.StatusCode) body, _ = io.ReadAll(resp.Body) resp.Body.Close() test.Equal(t, "", string(body)) test.Equal(t, "nsq; version=1.0", resp.Header.Get("X-NSQ-Content-Type")) _, err = topic.GetExistingChannel(channelName) test.NotNil(t, err) url = fmt.Sprintf("http://%s/topic/delete?topic=%s", httpAddr, topicName) resp, err = http.Post(url, "application/json", nil) test.Nil(t, err) test.Equal(t, 200, resp.StatusCode) body, _ = io.ReadAll(resp.Body) resp.Body.Close() test.Equal(t, "", string(body)) test.Equal(t, "nsq; version=1.0", resp.Header.Get("X-NSQ-Content-Type")) _, err = nsqd.GetExistingTopic(topicName) test.NotNil(t, err) } func TestHTTPClientStats(t *testing.T) { topicName := "test_http_client_stats" + strconv.Itoa(int(time.Now().Unix())) opts := NewOptions() opts.Logger = test.NewTestLogger(t) tcpAddr, httpAddr, nsqd := mustStartNSQD(opts) defer os.RemoveAll(opts.DataPath) defer nsqd.Exit() conn, err := mustConnectNSQD(tcpAddr) test.Nil(t, err) defer conn.Close() identify(t, conn, nil, frameTypeResponse) sub(t, conn, topicName, "ch") var d struct { Topics []struct { Channels []struct { ClientCount int `json:"client_count"` Clients []struct { } `json:"clients"` } `json:"channels"` } `json:"topics"` Memory *struct{} `json:"memory,omitempty"` } endpoint := fmt.Sprintf("http://%s/stats?format=json", httpAddr) err = http_api.NewClient(nil, ConnectTimeout, RequestTimeout).GETV1(endpoint, &d) test.Nil(t, err) test.Equal(t, 1, len(d.Topics[0].Channels[0].Clients)) test.Equal(t, 1, d.Topics[0].Channels[0].ClientCount) test.NotNil(t, d.Memory) endpoint = fmt.Sprintf("http://%s/stats?format=json&include_clients=true", httpAddr) err = http_api.NewClient(nil, ConnectTimeout, RequestTimeout).GETV1(endpoint, &d) test.Nil(t, err) test.Equal(t, 1, len(d.Topics[0].Channels[0].Clients)) test.Equal(t, 1, d.Topics[0].Channels[0].ClientCount) endpoint = fmt.Sprintf("http://%s/stats?format=json&include_clients=false", httpAddr) err = http_api.NewClient(nil, ConnectTimeout, RequestTimeout).GETV1(endpoint, &d) test.Nil(t, err) test.Equal(t, 0, len(d.Topics[0].Channels[0].Clients)) test.Equal(t, 1, d.Topics[0].Channels[0].ClientCount) endpoint = fmt.Sprintf("http://%s/stats?format=json&include_mem=true", httpAddr) err = http_api.NewClient(nil, ConnectTimeout, RequestTimeout).GETV1(endpoint, &d) test.Nil(t, err) test.NotNil(t, d.Memory) d.Memory = nil endpoint = fmt.Sprintf("http://%s/stats?format=json&include_mem=false", httpAddr) err = http_api.NewClient(nil, ConnectTimeout, RequestTimeout).GETV1(endpoint, &d) test.Nil(t, err) test.Nil(t, d.Memory) } func TestHTTPgetStatusJSON(t *testing.T) { testTime := time.Now() opts := NewOptions() opts.Logger = test.NewTestLogger(t) _, httpAddr, nsqd := mustStartNSQD(opts) defer os.RemoveAll(opts.DataPath) defer nsqd.Exit() nsqd.startTime = testTime expectedJSON := fmt.Sprintf(`{"version":"%v","health":"OK","start_time":%v,"topics":[],"memory":{`, version.Binary, testTime.Unix()) url := fmt.Sprintf("http://%s/stats?format=json", httpAddr) resp, err := http.Get(url) test.Nil(t, err) defer resp.Body.Close() body, _ := io.ReadAll(resp.Body) test.Equal(t, 200, resp.StatusCode) test.Equal(t, true, strings.HasPrefix(string(body), expectedJSON)) } func TestHTTPgetStatusText(t *testing.T) { testTime := time.Now() opts := NewOptions() opts.Logger = test.NewTestLogger(t) _, httpAddr, nsqd := mustStartNSQD(opts) defer os.RemoveAll(opts.DataPath) defer nsqd.Exit() nsqd.startTime = testTime url := fmt.Sprintf("http://%s/stats?format=text", httpAddr) resp, err := http.Get(url) test.Nil(t, err) defer resp.Body.Close() body, _ := io.ReadAll(resp.Body) test.Equal(t, 200, resp.StatusCode) test.NotNil(t, body) } func TestHTTPconfig(t *testing.T) { lopts := nsqlookupd.NewOptions() lopts.Logger = test.NewTestLogger(t) lopts1 := *lopts _, _, lookupd1 := mustStartNSQLookupd(&lopts1) defer lookupd1.Exit() lopts2 := *lopts _, _, lookupd2 := mustStartNSQLookupd(&lopts2) defer lookupd2.Exit() opts := NewOptions() opts.Logger = test.NewTestLogger(t) _, httpAddr, nsqd := mustStartNSQD(opts) defer os.RemoveAll(opts.DataPath) defer nsqd.Exit() url := fmt.Sprintf("http://%s/config/nsqlookupd_tcp_addresses", httpAddr) resp, err := http.Get(url) test.Nil(t, err) defer resp.Body.Close() body, _ := io.ReadAll(resp.Body) test.Equal(t, 200, resp.StatusCode) test.Equal(t, "[]", string(body)) client := http.Client{} addrs := fmt.Sprintf(`["%s","%s"]`, lookupd1.RealTCPAddr().String(), lookupd2.RealTCPAddr().String()) url = fmt.Sprintf("http://%s/config/nsqlookupd_tcp_addresses", httpAddr) req, err := http.NewRequest("PUT", url, bytes.NewBuffer([]byte(addrs))) test.Nil(t, err) resp, err = client.Do(req) test.Nil(t, err) defer resp.Body.Close() body, _ = io.ReadAll(resp.Body) test.Equal(t, 200, resp.StatusCode) test.Equal(t, addrs, string(body)) url = fmt.Sprintf("http://%s/config/log_level", httpAddr) req, err = http.NewRequest("PUT", url, bytes.NewBuffer([]byte(`fatal`))) test.Nil(t, err) resp, err = client.Do(req) test.Nil(t, err) defer resp.Body.Close() _, _ = io.ReadAll(resp.Body) test.Equal(t, 200, resp.StatusCode) test.Equal(t, LOG_FATAL, nsqd.getOpts().LogLevel) url = fmt.Sprintf("http://%s/config/log_level", httpAddr) req, err = http.NewRequest("PUT", url, bytes.NewBuffer([]byte(`bad`))) test.Nil(t, err) resp, err = client.Do(req) test.Nil(t, err) defer resp.Body.Close() _, _ = io.ReadAll(resp.Body) test.Equal(t, 400, resp.StatusCode) } func TestHTTPerrors(t *testing.T) { opts := NewOptions() opts.Logger = test.NewTestLogger(t) _, httpAddr, nsqd := mustStartNSQD(opts) defer os.RemoveAll(opts.DataPath) defer nsqd.Exit() url := fmt.Sprintf("http://%s/stats", httpAddr) resp, err := http.Post(url, "text/plain", nil) test.Nil(t, err) defer resp.Body.Close() body, _ := io.ReadAll(resp.Body) test.Equal(t, 405, resp.StatusCode) test.Equal(t, `{"message":"METHOD_NOT_ALLOWED"}`, string(body)) url = fmt.Sprintf("http://%s/not_found", httpAddr) resp, err = http.Get(url) test.Nil(t, err) defer resp.Body.Close() body, _ = io.ReadAll(resp.Body) test.Equal(t, 404, resp.StatusCode) test.Equal(t, `{"message":"NOT_FOUND"}`, string(body)) } func TestDeleteTopic(t *testing.T) { opts := NewOptions() opts.Logger = test.NewTestLogger(t) _, httpAddr, nsqd := mustStartNSQD(opts) defer os.RemoveAll(opts.DataPath) defer nsqd.Exit() em := ErrMessage{} url := fmt.Sprintf("http://%s/topic/delete", httpAddr) resp, err := http.Post(url, "application/json", nil) test.Nil(t, err) test.Equal(t, 400, resp.StatusCode) test.Equal(t, "Bad Request", http.StatusText(resp.StatusCode)) body, _ := io.ReadAll(resp.Body) resp.Body.Close() t.Logf("%s", body) err = json.Unmarshal(body, &em) test.Nil(t, err) test.Equal(t, "MISSING_ARG_TOPIC", em.Message) topicName := "test_http_delete_topic" + strconv.Itoa(int(time.Now().Unix())) url = fmt.Sprintf("http://%s/topic/delete?topic=%s", httpAddr, topicName) resp, err = http.Post(url, "application/json", nil) test.Nil(t, err) test.Equal(t, 404, resp.StatusCode) test.Equal(t, "Not Found", http.StatusText(resp.StatusCode)) body, _ = io.ReadAll(resp.Body) resp.Body.Close() t.Logf("%s", body) err = json.Unmarshal(body, &em) test.Nil(t, err) test.Equal(t, "TOPIC_NOT_FOUND", em.Message) nsqd.GetTopic(topicName) resp, err = http.Post(url, "application/json", nil) test.Nil(t, err) test.Equal(t, 200, resp.StatusCode) body, _ = io.ReadAll(resp.Body) resp.Body.Close() t.Logf("%s", body) test.Equal(t, []byte(""), body) } func TestEmptyTopic(t *testing.T) { opts := NewOptions() opts.Logger = test.NewTestLogger(t) _, httpAddr, nsqd := mustStartNSQD(opts) defer os.RemoveAll(opts.DataPath) defer nsqd.Exit() em := ErrMessage{} url := fmt.Sprintf("http://%s/topic/empty", httpAddr) resp, err := http.Post(url, "application/json", nil) test.Nil(t, err) test.Equal(t, 400, resp.StatusCode) test.Equal(t, "Bad Request", http.StatusText(resp.StatusCode)) body, _ := io.ReadAll(resp.Body) resp.Body.Close() t.Logf("%s", body) err = json.Unmarshal(body, &em) test.Nil(t, err) test.Equal(t, "MISSING_ARG_TOPIC", em.Message) topicName := "test_http_empty_topic" + strconv.Itoa(int(time.Now().Unix())) url = fmt.Sprintf("http://%s/topic/empty?topic=%s", httpAddr, topicName+"$") resp, err = http.Post(url, "application/json", nil) test.Nil(t, err) test.Equal(t, 400, resp.StatusCode) test.Equal(t, "Bad Request", http.StatusText(resp.StatusCode)) body, _ = io.ReadAll(resp.Body) resp.Body.Close() t.Logf("%s", body) err = json.Unmarshal(body, &em) test.Nil(t, err) test.Equal(t, "INVALID_TOPIC", em.Message) url = fmt.Sprintf("http://%s/topic/empty?topic=%s", httpAddr, topicName) resp, err = http.Post(url, "application/json", nil) test.Nil(t, err) test.Equal(t, 404, resp.StatusCode) test.Equal(t, "Not Found", http.StatusText(resp.StatusCode)) body, _ = io.ReadAll(resp.Body) resp.Body.Close() t.Logf("%s", body) err = json.Unmarshal(body, &em) test.Nil(t, err) test.Equal(t, "TOPIC_NOT_FOUND", em.Message) nsqd.GetTopic(topicName) resp, err = http.Post(url, "application/json", nil) test.Nil(t, err) test.Equal(t, 200, resp.StatusCode) body, _ = io.ReadAll(resp.Body) resp.Body.Close() t.Logf("%s", body) test.Equal(t, []byte(""), body) } func TestEmptyChannel(t *testing.T) { opts := NewOptions() opts.Logger = test.NewTestLogger(t) _, httpAddr, nsqd := mustStartNSQD(opts) defer os.RemoveAll(opts.DataPath) defer nsqd.Exit() em := ErrMessage{} url := fmt.Sprintf("http://%s/channel/empty", httpAddr) resp, err := http.Post(url, "application/json", nil) test.Nil(t, err) test.Equal(t, 400, resp.StatusCode) test.Equal(t, "Bad Request", http.StatusText(resp.StatusCode)) body, _ := io.ReadAll(resp.Body) resp.Body.Close() t.Logf("%s", body) err = json.Unmarshal(body, &em) test.Nil(t, err) test.Equal(t, "MISSING_ARG_TOPIC", em.Message) topicName := "test_http_empty_channel" + strconv.Itoa(int(time.Now().Unix())) url = fmt.Sprintf("http://%s/channel/empty?topic=%s", httpAddr, topicName) resp, err = http.Post(url, "application/json", nil) test.Nil(t, err) test.Equal(t, 400, resp.StatusCode) test.Equal(t, "Bad Request", http.StatusText(resp.StatusCode)) body, _ = io.ReadAll(resp.Body) resp.Body.Close() t.Logf("%s", body) err = json.Unmarshal(body, &em) test.Nil(t, err) test.Equal(t, "MISSING_ARG_CHANNEL", em.Message) channelName := "ch" url = fmt.Sprintf("http://%s/channel/empty?topic=%s&channel=%s", httpAddr, topicName, channelName) resp, err = http.Post(url, "application/json", nil) test.Nil(t, err) test.Equal(t, 404, resp.StatusCode) test.Equal(t, "Not Found", http.StatusText(resp.StatusCode)) body, _ = io.ReadAll(resp.Body) resp.Body.Close() t.Logf("%s", body) err = json.Unmarshal(body, &em) test.Nil(t, err) test.Equal(t, "TOPIC_NOT_FOUND", em.Message) topic := nsqd.GetTopic(topicName) url = fmt.Sprintf("http://%s/channel/empty?topic=%s&channel=%s", httpAddr, topicName, channelName) resp, err = http.Post(url, "application/json", nil) test.Nil(t, err) test.Equal(t, 404, resp.StatusCode) test.Equal(t, "Not Found", http.StatusText(resp.StatusCode)) body, _ = io.ReadAll(resp.Body) resp.Body.Close() t.Logf("%s", body) err = json.Unmarshal(body, &em) test.Nil(t, err) test.Equal(t, "CHANNEL_NOT_FOUND", em.Message) topic.GetChannel(channelName) resp, err = http.Post(url, "application/json", nil) test.Nil(t, err) test.Equal(t, 200, resp.StatusCode) body, _ = io.ReadAll(resp.Body) resp.Body.Close() t.Logf("%s", body) test.Equal(t, []byte(""), body) } func TestInfo(t *testing.T) { opts := NewOptions() opts.Logger = test.NewTestLogger(t) _, httpAddr, nsqd := mustStartNSQD(opts) defer os.RemoveAll(opts.DataPath) defer nsqd.Exit() info := InfoDoc{} url := fmt.Sprintf("http://%s/info", httpAddr) resp, err := http.Get(url) test.Nil(t, err) test.Equal(t, 200, resp.StatusCode) body, _ := io.ReadAll(resp.Body) resp.Body.Close() t.Logf("%s", body) err = json.Unmarshal(body, &info) test.Nil(t, err) test.Equal(t, version.Binary, info.Version) } func BenchmarkHTTPpub(b *testing.B) { var wg sync.WaitGroup b.StopTimer() opts := NewOptions() opts.Logger = test.NewTestLogger(b) opts.MemQueueSize = int64(b.N) _, httpAddr, nsqd := mustStartNSQD(opts) defer os.RemoveAll(opts.DataPath) msg := make([]byte, 256) topicName := "bench_http_pub" + strconv.Itoa(int(time.Now().Unix())) url := fmt.Sprintf("http://%s/pub?topic=%s", httpAddr, topicName) client := &http.Client{} b.SetBytes(int64(len(msg))) b.StartTimer() for j := 0; j < runtime.GOMAXPROCS(0); j++ { wg.Add(1) go func() { num := b.N / runtime.GOMAXPROCS(0) for i := 0; i < num; i++ { buf := bytes.NewBuffer(msg) req, _ := http.NewRequest("POST", url, buf) resp, err := client.Do(req) if err != nil { panic(err.Error()) } body, _ := io.ReadAll(resp.Body) if !bytes.Equal(body, []byte("OK")) { panic("bad response") } resp.Body.Close() } wg.Done() }() } wg.Wait() b.StopTimer() nsqd.Exit() } ================================================ FILE: nsqd/in_flight_pqueue.go ================================================ package nsqd type inFlightPqueue []*Message func newInFlightPqueue(capacity int) inFlightPqueue { return make(inFlightPqueue, 0, capacity) } func (pq inFlightPqueue) Swap(i, j int) { pq[i], pq[j] = pq[j], pq[i] pq[i].index = i pq[j].index = j } func (pq *inFlightPqueue) Push(x *Message) { n := len(*pq) c := cap(*pq) if n+1 > c { npq := make(inFlightPqueue, n, c*2) copy(npq, *pq) *pq = npq } *pq = (*pq)[0 : n+1] x.index = n (*pq)[n] = x pq.up(n) } func (pq *inFlightPqueue) Pop() *Message { n := len(*pq) c := cap(*pq) pq.Swap(0, n-1) pq.down(0, n-1) if n < (c/2) && c > 25 { npq := make(inFlightPqueue, n, c/2) copy(npq, *pq) *pq = npq } x := (*pq)[n-1] x.index = -1 *pq = (*pq)[0 : n-1] return x } func (pq *inFlightPqueue) Remove(i int) *Message { n := len(*pq) if n-1 != i { pq.Swap(i, n-1) pq.down(i, n-1) pq.up(i) } x := (*pq)[n-1] x.index = -1 *pq = (*pq)[0 : n-1] return x } func (pq *inFlightPqueue) PeekAndShift(max int64) (*Message, int64) { if len(*pq) == 0 { return nil, 0 } x := (*pq)[0] if x.pri > max { return nil, x.pri - max } pq.Pop() return x, 0 } func (pq *inFlightPqueue) up(j int) { for { i := (j - 1) / 2 // parent if i == j || (*pq)[j].pri >= (*pq)[i].pri { break } pq.Swap(i, j) j = i } } func (pq *inFlightPqueue) down(i, n int) { for { j1 := 2*i + 1 if j1 >= n || j1 < 0 { // j1 < 0 after int overflow break } j := j1 // left child if j2 := j1 + 1; j2 < n && (*pq)[j1].pri >= (*pq)[j2].pri { j = j2 // = 2*i + 2 // right child } if (*pq)[j].pri >= (*pq)[i].pri { break } pq.Swap(i, j) i = j } } ================================================ FILE: nsqd/in_flight_pqueue_test.go ================================================ package nsqd import ( "fmt" "math/rand" "sort" "testing" "github.com/nsqio/nsq/internal/test" ) func TestPriorityQueue(t *testing.T) { c := 100 pq := newInFlightPqueue(c) for i := 0; i < c+1; i++ { pq.Push(&Message{clientID: int64(i), pri: int64(i)}) } test.Equal(t, c+1, len(pq)) test.Equal(t, c*2, cap(pq)) for i := 0; i < c+1; i++ { msg := pq.Pop() test.Equal(t, int64(i), msg.clientID) } test.Equal(t, c/4, cap(pq)) } func TestUnsortedInsert(t *testing.T) { c := 100 pq := newInFlightPqueue(c) ints := make([]int, 0, c) for i := 0; i < c; i++ { v := rand.Int() ints = append(ints, v) pq.Push(&Message{pri: int64(v)}) } test.Equal(t, c, len(pq)) test.Equal(t, c, cap(pq)) sort.Ints(ints) for i := 0; i < c; i++ { msg, _ := pq.PeekAndShift(int64(ints[len(ints)-1])) test.Equal(t, int64(ints[i]), msg.pri) } } func TestRemove(t *testing.T) { c := 100 pq := newInFlightPqueue(c) msgs := make(map[MessageID]*Message) for i := 0; i < c; i++ { m := &Message{pri: int64(rand.Intn(100000000))} copy(m.ID[:], fmt.Sprintf("%016d", m.pri)) msgs[m.ID] = m pq.Push(m) } for i := 0; i < 10; i++ { idx := rand.Intn((c - 1) - i) var fm *Message for _, m := range msgs { if m.index == idx { fm = m break } } rm := pq.Remove(idx) test.Equal(t, fmt.Sprintf("%s", fm.ID), fmt.Sprintf("%s", rm.ID)) } lastPriority := pq.Pop().pri for i := 0; i < (c - 10 - 1); i++ { msg := pq.Pop() test.Equal(t, true, lastPriority <= msg.pri) lastPriority = msg.pri } } ================================================ FILE: nsqd/logger.go ================================================ package nsqd import ( "github.com/nsqio/nsq/internal/lg" ) type Logger lg.Logger const ( LOG_DEBUG = lg.DEBUG LOG_INFO = lg.INFO LOG_WARN = lg.WARN LOG_ERROR = lg.ERROR LOG_FATAL = lg.FATAL ) func (n *NSQD) logf(level lg.LogLevel, f string, args ...interface{}) { opts := n.getOpts() lg.Logf(opts.Logger, opts.LogLevel, level, f, args...) } ================================================ FILE: nsqd/lookup.go ================================================ package nsqd import ( "bytes" "encoding/json" "net" "os" "strconv" "time" "github.com/nsqio/go-nsq" "github.com/nsqio/nsq/internal/version" ) func connectCallback(n *NSQD, hostname string) func(*lookupPeer) { return func(lp *lookupPeer) { ci := make(map[string]interface{}) ci["version"] = version.Binary ci["tcp_port"] = n.getOpts().BroadcastTCPPort ci["http_port"] = n.getOpts().BroadcastHTTPPort ci["hostname"] = hostname ci["broadcast_address"] = n.getOpts().BroadcastAddress ci["topology_zone"] = n.getOpts().TopologyZone ci["topology_region"] = n.getOpts().TopologyRegion cmd, err := nsq.Identify(ci) if err != nil { lp.Close() return } resp, err := lp.Command(cmd) if err != nil { n.logf(LOG_ERROR, "LOOKUPD(%s): %s - %s", lp, cmd, err) return } else if bytes.Equal(resp, []byte("E_INVALID")) { n.logf(LOG_INFO, "LOOKUPD(%s): lookupd returned %s", lp, resp) lp.Close() return } err = json.Unmarshal(resp, &lp.Info) if err != nil { n.logf(LOG_ERROR, "LOOKUPD(%s): parsing response - %s", lp, resp) lp.Close() return } n.logf(LOG_INFO, "LOOKUPD(%s): peer info %+v", lp, lp.Info) if lp.Info.BroadcastAddress == "" { n.logf(LOG_ERROR, "LOOKUPD(%s): no broadcast address", lp) } // build all the commands first so we exit the lock(s) as fast as possible var commands []*nsq.Command n.RLock() for _, topic := range n.topicMap { topic.RLock() if len(topic.channelMap) == 0 { commands = append(commands, nsq.Register(topic.name, "")) } else { for _, channel := range topic.channelMap { commands = append(commands, nsq.Register(channel.topicName, channel.name)) } } topic.RUnlock() } n.RUnlock() for _, cmd := range commands { n.logf(LOG_INFO, "LOOKUPD(%s): %s", lp, cmd) _, err := lp.Command(cmd) if err != nil { n.logf(LOG_ERROR, "LOOKUPD(%s): %s - %s", lp, cmd, err) return } } } } func (n *NSQD) lookupLoop() { var lookupPeers []*lookupPeer var lookupAddrs []string connect := true hostname, err := os.Hostname() if err != nil { n.logf(LOG_FATAL, "failed to get hostname - %s", err) os.Exit(1) } // for announcements, lookupd determines the host automatically ticker := time.NewTicker(15 * time.Second) defer ticker.Stop() for { if connect { for _, host := range n.getOpts().NSQLookupdTCPAddresses { if in(host, lookupAddrs) { continue } n.logf(LOG_INFO, "LOOKUP(%s): adding peer", host) lookupPeer := newLookupPeer(host, n.getOpts().MaxBodySize, n.logf, connectCallback(n, hostname)) lookupPeer.Command(nil) // start the connection lookupPeers = append(lookupPeers, lookupPeer) lookupAddrs = append(lookupAddrs, host) } n.lookupPeers.Store(lookupPeers) connect = false } select { case <-ticker.C: // send a heartbeat and read a response (read detects closed conns) for _, lookupPeer := range lookupPeers { n.logf(LOG_DEBUG, "LOOKUPD(%s): sending heartbeat", lookupPeer) cmd := nsq.Ping() _, err := lookupPeer.Command(cmd) if err != nil { n.logf(LOG_ERROR, "LOOKUPD(%s): %s - %s", lookupPeer, cmd, err) } } case val := <-n.notifyChan: var cmd *nsq.Command var branch string switch val := val.(type) { case *Channel: // notify all nsqlookupds that a new channel exists, or that it's removed branch = "channel" channel := val if channel.Exiting() { cmd = nsq.UnRegister(channel.topicName, channel.name) } else { cmd = nsq.Register(channel.topicName, channel.name) } case *Topic: // notify all nsqlookupds that a new topic exists, or that it's removed branch = "topic" topic := val if topic.Exiting() { cmd = nsq.UnRegister(topic.name, "") } else { cmd = nsq.Register(topic.name, "") } } for _, lookupPeer := range lookupPeers { n.logf(LOG_INFO, "LOOKUPD(%s): %s %s", lookupPeer, branch, cmd) _, err := lookupPeer.Command(cmd) if err != nil { n.logf(LOG_ERROR, "LOOKUPD(%s): %s - %s", lookupPeer, cmd, err) } } case <-n.optsNotificationChan: var tmpPeers []*lookupPeer var tmpAddrs []string for _, lp := range lookupPeers { if in(lp.addr, n.getOpts().NSQLookupdTCPAddresses) { tmpPeers = append(tmpPeers, lp) tmpAddrs = append(tmpAddrs, lp.addr) continue } n.logf(LOG_INFO, "LOOKUP(%s): removing peer", lp) lp.Close() } lookupPeers = tmpPeers lookupAddrs = tmpAddrs connect = true case <-n.exitChan: goto exit } } exit: n.logf(LOG_INFO, "LOOKUP: closing") } func in(s string, lst []string) bool { for _, v := range lst { if s == v { return true } } return false } func (n *NSQD) lookupdHTTPAddrs() []string { var lookupHTTPAddrs []string lookupPeers := n.lookupPeers.Load() if lookupPeers == nil { return nil } for _, lp := range lookupPeers.([]*lookupPeer) { if len(lp.Info.BroadcastAddress) <= 0 { continue } addr := net.JoinHostPort(lp.Info.BroadcastAddress, strconv.Itoa(lp.Info.HTTPPort)) lookupHTTPAddrs = append(lookupHTTPAddrs, addr) } return lookupHTTPAddrs } ================================================ FILE: nsqd/lookup_peer.go ================================================ package nsqd import ( "encoding/binary" "fmt" "io" "net" "time" "github.com/nsqio/go-nsq" "github.com/nsqio/nsq/internal/lg" ) // lookupPeer is a low-level type for connecting/reading/writing to nsqlookupd // // A lookupPeer instance is designed to connect lazily to nsqlookupd and reconnect // gracefully (i.e. it is all handled by the library). Clients can simply use the // Command interface to perform a round-trip. type lookupPeer struct { logf lg.AppLogFunc addr string conn net.Conn state int32 connectCallback func(*lookupPeer) maxBodySize int64 Info peerInfo } // peerInfo contains metadata for a lookupPeer instance (and is JSON marshalable) type peerInfo struct { TCPPort int `json:"tcp_port"` HTTPPort int `json:"http_port"` Version string `json:"version"` BroadcastAddress string `json:"broadcast_address"` } // newLookupPeer creates a new lookupPeer instance connecting to the supplied address. // // The supplied connectCallback will be called *every* time the instance connects. func newLookupPeer(addr string, maxBodySize int64, l lg.AppLogFunc, connectCallback func(*lookupPeer)) *lookupPeer { return &lookupPeer{ logf: l, addr: addr, state: stateDisconnected, maxBodySize: maxBodySize, connectCallback: connectCallback, } } // Connect will Dial the specified address, with timeouts func (lp *lookupPeer) Connect() error { lp.logf(lg.INFO, "LOOKUP connecting to %s", lp.addr) conn, err := net.DialTimeout("tcp", lp.addr, time.Second) if err != nil { return err } lp.conn = conn return nil } // String returns the specified address func (lp *lookupPeer) String() string { return lp.addr } // Read implements the io.Reader interface, adding deadlines func (lp *lookupPeer) Read(data []byte) (int, error) { lp.conn.SetReadDeadline(time.Now().Add(time.Second)) return lp.conn.Read(data) } // Write implements the io.Writer interface, adding deadlines func (lp *lookupPeer) Write(data []byte) (int, error) { lp.conn.SetWriteDeadline(time.Now().Add(time.Second)) return lp.conn.Write(data) } // Close implements the io.Closer interface func (lp *lookupPeer) Close() error { lp.state = stateDisconnected if lp.conn != nil { return lp.conn.Close() } return nil } // Command performs a round-trip for the specified Command. // // It will lazily connect to nsqlookupd and gracefully handle // reconnecting in the event of a failure. // // It returns the response from nsqlookupd as []byte func (lp *lookupPeer) Command(cmd *nsq.Command) ([]byte, error) { initialState := lp.state if lp.state != stateConnected { err := lp.Connect() if err != nil { return nil, err } lp.state = stateConnected _, err = lp.Write(nsq.MagicV1) if err != nil { lp.Close() return nil, err } if initialState == stateDisconnected { lp.connectCallback(lp) } if lp.state != stateConnected { return nil, fmt.Errorf("lookupPeer connectCallback() failed") } } if cmd == nil { return nil, nil } _, err := cmd.WriteTo(lp) if err != nil { lp.Close() return nil, err } resp, err := readResponseBounded(lp, lp.maxBodySize) if err != nil { lp.Close() return nil, err } return resp, nil } func readResponseBounded(r io.Reader, limit int64) ([]byte, error) { var msgSize int32 // message size err := binary.Read(r, binary.BigEndian, &msgSize) if err != nil { return nil, err } if int64(msgSize) > limit { return nil, fmt.Errorf("response body size (%d) is greater than limit (%d)", msgSize, limit) } // message binary data buf := make([]byte, msgSize) _, err = io.ReadFull(r, buf) if err != nil { return nil, err } return buf, nil } ================================================ FILE: nsqd/message.go ================================================ package nsqd import ( "encoding/binary" "fmt" "io" "time" ) const ( MsgIDLength = 16 minValidMsgLength = MsgIDLength + 8 + 2 // Timestamp + Attempts ) type MessageID [MsgIDLength]byte type Message struct { ID MessageID Body []byte Timestamp int64 Attempts uint16 // for in-flight handling deliveryTS time.Time clientID int64 pri int64 index int deferred time.Duration } func NewMessage(id MessageID, body []byte) *Message { return &Message{ ID: id, Body: body, Timestamp: time.Now().UnixNano(), } } func (m *Message) WriteTo(w io.Writer) (int64, error) { var buf [10]byte var total int64 binary.BigEndian.PutUint64(buf[:8], uint64(m.Timestamp)) binary.BigEndian.PutUint16(buf[8:10], uint16(m.Attempts)) n, err := w.Write(buf[:]) total += int64(n) if err != nil { return total, err } n, err = w.Write(m.ID[:]) total += int64(n) if err != nil { return total, err } n, err = w.Write(m.Body) total += int64(n) if err != nil { return total, err } return total, nil } // decodeMessage deserializes data (as []byte) and creates a new Message // // [x][x][x][x][x][x][x][x][x][x][x][x][x][x][x][x][x][x][x][x][x][x][x][x][x][x][x][x][x][x]... // | (int64) || || (hex string encoded in ASCII) || (binary) // | 8-byte || || 16-byte || N-byte // ------------------------------------------------------------------------------------------... // nanosecond timestamp ^^ message ID message body // (uint16) // 2-byte // attempts func decodeMessage(b []byte) (*Message, error) { var msg Message if len(b) < minValidMsgLength { return nil, fmt.Errorf("invalid message buffer size (%d)", len(b)) } msg.Timestamp = int64(binary.BigEndian.Uint64(b[:8])) msg.Attempts = binary.BigEndian.Uint16(b[8:10]) copy(msg.ID[:], b[10:10+MsgIDLength]) msg.Body = b[10+MsgIDLength:] return &msg, nil } func writeMessageToBackend(msg *Message, bq BackendQueue) error { buf := bufferPoolGet() defer bufferPoolPut(buf) _, err := msg.WriteTo(buf) if err != nil { return err } return bq.Put(buf.Bytes()) } ================================================ FILE: nsqd/nsqd.go ================================================ package nsqd import ( "context" "crypto/tls" "crypto/x509" "encoding/json" "errors" "fmt" "log" "math/rand" "net" "os" "path" "strings" "sync" "sync/atomic" "time" "github.com/nsqio/nsq/internal/clusterinfo" "github.com/nsqio/nsq/internal/dirlock" "github.com/nsqio/nsq/internal/http_api" "github.com/nsqio/nsq/internal/protocol" "github.com/nsqio/nsq/internal/statsd" "github.com/nsqio/nsq/internal/util" "github.com/nsqio/nsq/internal/version" ) const ( TLSNotRequired = iota TLSRequiredExceptHTTP TLSRequired ) type errStore struct { err error } type NSQD struct { // 64bit atomic vars need to be first for proper alignment on 32bit platforms clientIDSequence int64 sync.RWMutex ctx context.Context // ctxCancel cancels a context that main() is waiting on ctxCancel context.CancelFunc opts atomic.Value dl *dirlock.DirLock isLoading int32 isExiting int32 errValue atomic.Value startTime time.Time topicMap map[string]*Topic lookupPeers atomic.Value tcpServer *tcpServer tcpListener net.Listener httpListener net.Listener httpsListener net.Listener tlsConfig *tls.Config clientTLSConfig *tls.Config poolSize int notifyChan chan interface{} optsNotificationChan chan struct{} exitChan chan int waitGroup util.WaitGroupWrapper ci *clusterinfo.ClusterInfo } func New(opts *Options) (*NSQD, error) { var err error dataPath := opts.DataPath if opts.DataPath == "" { cwd, _ := os.Getwd() dataPath = cwd } if opts.Logger == nil { opts.Logger = log.New(os.Stderr, opts.LogPrefix, log.Ldate|log.Ltime|log.Lmicroseconds) } n := &NSQD{ startTime: time.Now(), topicMap: make(map[string]*Topic), exitChan: make(chan int), notifyChan: make(chan interface{}), optsNotificationChan: make(chan struct{}, 1), dl: dirlock.New(dataPath), } n.ctx, n.ctxCancel = context.WithCancel(context.Background()) httpcli := http_api.NewClient(nil, opts.HTTPClientConnectTimeout, opts.HTTPClientRequestTimeout) n.ci = clusterinfo.New(n.logf, httpcli) n.lookupPeers.Store([]*lookupPeer{}) n.swapOpts(opts) n.errValue.Store(errStore{}) err = n.dl.Lock() if err != nil { return nil, fmt.Errorf("failed to lock data-path: %v", err) } if opts.MaxDeflateLevel < 1 || opts.MaxDeflateLevel > 9 { return nil, errors.New("--max-deflate-level must be [1,9]") } if opts.ID < 0 || opts.ID >= 1024 { return nil, errors.New("--node-id must be [0,1024)") } if opts.TLSClientAuthPolicy != "" && opts.TLSRequired == TLSNotRequired { opts.TLSRequired = TLSRequired } tlsConfig, err := buildTLSConfig(opts) if err != nil { return nil, fmt.Errorf("failed to build TLS config - %s", err) } if tlsConfig == nil && opts.TLSRequired != TLSNotRequired { return nil, errors.New("cannot require TLS client connections without TLS key and cert") } n.tlsConfig = tlsConfig clientTLSConfig, err := buildClientTLSConfig(opts) if err != nil { return nil, fmt.Errorf("failed to build client TLS config - %s", err) } n.clientTLSConfig = clientTLSConfig if opts.AuthHTTPRequestMethod != "post" && opts.AuthHTTPRequestMethod != "get" { return nil, errors.New("--auth-http-request-method must be post or get") } for _, v := range opts.E2EProcessingLatencyPercentiles { if v <= 0 || v > 1 { return nil, fmt.Errorf("invalid E2E processing latency percentile: %v", v) } } n.logf(LOG_INFO, version.String("nsqd")) n.logf(LOG_INFO, "ID: %d", opts.ID) n.tcpServer = &tcpServer{nsqd: n} n.tcpListener, err = net.Listen(util.TypeOfAddr(opts.TCPAddress), opts.TCPAddress) if err != nil { return nil, fmt.Errorf("listen (%s) failed - %s", opts.TCPAddress, err) } if opts.HTTPAddress != "" { n.httpListener, err = net.Listen(util.TypeOfAddr(opts.HTTPAddress), opts.HTTPAddress) if err != nil { return nil, fmt.Errorf("listen (%s) failed - %s", opts.HTTPAddress, err) } } if n.tlsConfig != nil && opts.HTTPSAddress != "" { n.httpsListener, err = tls.Listen("tcp", opts.HTTPSAddress, n.tlsConfig) if err != nil { return nil, fmt.Errorf("listen (%s) failed - %s", opts.HTTPSAddress, err) } } if opts.BroadcastHTTPPort == 0 { tcpAddr, ok := n.RealHTTPAddr().(*net.TCPAddr) if ok { opts.BroadcastHTTPPort = tcpAddr.Port } } if opts.BroadcastTCPPort == 0 { tcpAddr, ok := n.RealTCPAddr().(*net.TCPAddr) if ok { opts.BroadcastTCPPort = tcpAddr.Port } } if opts.StatsdPrefix != "" { var port string = fmt.Sprint(opts.BroadcastHTTPPort) statsdHostKey := statsd.HostKey(net.JoinHostPort(opts.BroadcastAddress, port)) prefixWithHost := strings.Replace(opts.StatsdPrefix, "%s", statsdHostKey, -1) if prefixWithHost[len(prefixWithHost)-1] != '.' { prefixWithHost += "." } opts.StatsdPrefix = prefixWithHost } return n, nil } func (n *NSQD) getOpts() *Options { return n.opts.Load().(*Options) } func (n *NSQD) swapOpts(opts *Options) { n.opts.Store(opts) } func (n *NSQD) triggerOptsNotification() { select { case n.optsNotificationChan <- struct{}{}: default: } } func (n *NSQD) RealTCPAddr() net.Addr { if n.tcpListener == nil { return &net.TCPAddr{} } return n.tcpListener.Addr() } func (n *NSQD) RealHTTPAddr() net.Addr { if n.httpListener == nil { return &net.TCPAddr{} } return n.httpListener.Addr() } func (n *NSQD) RealHTTPSAddr() *net.TCPAddr { if n.httpsListener == nil { return &net.TCPAddr{} } return n.httpsListener.Addr().(*net.TCPAddr) } func (n *NSQD) SetHealth(err error) { n.errValue.Store(errStore{err: err}) } func (n *NSQD) IsHealthy() bool { return n.GetError() == nil } func (n *NSQD) GetError() error { errValue := n.errValue.Load() return errValue.(errStore).err } func (n *NSQD) GetHealth() string { err := n.GetError() if err != nil { return fmt.Sprintf("NOK - %s", err) } return "OK" } func (n *NSQD) GetStartTime() time.Time { return n.startTime } func (n *NSQD) Main() error { exitCh := make(chan error) var once sync.Once exitFunc := func(err error) { once.Do(func() { if err != nil { n.logf(LOG_FATAL, "%s", err) } exitCh <- err }) } n.waitGroup.Wrap(func() { exitFunc(protocol.TCPServer(n.tcpListener, n.tcpServer, n.logf)) }) if n.httpListener != nil { httpServer := newHTTPServer(n, false, n.getOpts().TLSRequired == TLSRequired) n.waitGroup.Wrap(func() { exitFunc(http_api.Serve(n.httpListener, httpServer, "HTTP", n.logf)) }) } if n.httpsListener != nil { httpsServer := newHTTPServer(n, true, true) n.waitGroup.Wrap(func() { exitFunc(http_api.Serve(n.httpsListener, httpsServer, "HTTPS", n.logf)) }) } n.waitGroup.Wrap(n.queueScanLoop) n.waitGroup.Wrap(n.lookupLoop) if n.getOpts().StatsdAddress != "" { n.waitGroup.Wrap(n.statsdLoop) } err := <-exitCh return err } // Metadata is the collection of persistent information about the current NSQD. type Metadata struct { Topics []TopicMetadata `json:"topics"` Version string `json:"version"` } // TopicMetadata is the collection of persistent information about a topic. type TopicMetadata struct { Name string `json:"name"` Paused bool `json:"paused"` Channels []ChannelMetadata `json:"channels"` } // ChannelMetadata is the collection of persistent information about a channel. type ChannelMetadata struct { Name string `json:"name"` Paused bool `json:"paused"` } func newMetadataFile(opts *Options) string { return path.Join(opts.DataPath, "nsqd.dat") } func readOrEmpty(fn string) ([]byte, error) { data, err := os.ReadFile(fn) if err != nil { if !os.IsNotExist(err) { return nil, fmt.Errorf("failed to read metadata from %s - %s", fn, err) } } return data, nil } func writeSyncFile(fn string, data []byte) error { f, err := os.OpenFile(fn, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0600) if err != nil { return err } _, err = f.Write(data) if err == nil { err = f.Sync() } f.Close() return err } func (n *NSQD) LoadMetadata() error { atomic.StoreInt32(&n.isLoading, 1) defer atomic.StoreInt32(&n.isLoading, 0) fn := newMetadataFile(n.getOpts()) data, err := readOrEmpty(fn) if err != nil { return err } if data == nil { return nil // fresh start } var m Metadata err = json.Unmarshal(data, &m) if err != nil { return fmt.Errorf("failed to parse metadata in %s - %s", fn, err) } for _, t := range m.Topics { if !protocol.IsValidTopicName(t.Name) { n.logf(LOG_WARN, "skipping creation of invalid topic %s", t.Name) continue } topic := n.GetTopic(t.Name) if t.Paused { topic.Pause() } for _, c := range t.Channels { if !protocol.IsValidChannelName(c.Name) { n.logf(LOG_WARN, "skipping creation of invalid channel %s", c.Name) continue } channel := topic.GetChannel(c.Name) if c.Paused { channel.Pause() } } topic.Start() } return nil } // GetMetadata retrieves the current topic and channel set of the NSQ daemon. If // the ephemeral flag is set, ephemeral topics are also returned even though these // are not saved to disk. func (n *NSQD) GetMetadata(ephemeral bool) *Metadata { meta := &Metadata{ Version: version.Binary, } for _, topic := range n.topicMap { if topic.ephemeral && !ephemeral { continue } topicData := TopicMetadata{ Name: topic.name, Paused: topic.IsPaused(), } topic.Lock() for _, channel := range topic.channelMap { if channel.ephemeral { continue } topicData.Channels = append(topicData.Channels, ChannelMetadata{ Name: channel.name, Paused: channel.IsPaused(), }) } topic.Unlock() meta.Topics = append(meta.Topics, topicData) } return meta } func (n *NSQD) PersistMetadata() error { // persist metadata about what topics/channels we have, across restarts fileName := newMetadataFile(n.getOpts()) n.logf(LOG_INFO, "NSQ: persisting topic/channel metadata to %s", fileName) data, err := json.Marshal(n.GetMetadata(false)) if err != nil { return err } tmpFileName := fmt.Sprintf("%s.%d.tmp", fileName, rand.Int()) err = writeSyncFile(tmpFileName, data) if err != nil { return err } err = os.Rename(tmpFileName, fileName) if err != nil { return err } // technically should fsync DataPath here return nil } func (n *NSQD) Exit() { if !atomic.CompareAndSwapInt32(&n.isExiting, 0, 1) { // avoid double call return } if n.tcpListener != nil { n.tcpListener.Close() } if n.tcpServer != nil { n.tcpServer.Close() } if n.httpListener != nil { n.httpListener.Close() } if n.httpsListener != nil { n.httpsListener.Close() } n.Lock() err := n.PersistMetadata() if err != nil { n.logf(LOG_ERROR, "failed to persist metadata - %s", err) } n.logf(LOG_INFO, "NSQ: closing topics") for _, topic := range n.topicMap { topic.Close() } n.Unlock() n.logf(LOG_INFO, "NSQ: stopping subsystems") close(n.exitChan) n.waitGroup.Wait() n.dl.Unlock() n.logf(LOG_INFO, "NSQ: bye") n.ctxCancel() } // GetTopic performs a thread safe operation // to return a pointer to a Topic object (potentially new) func (n *NSQD) GetTopic(topicName string) *Topic { // most likely we already have this topic, so try read lock first n.RLock() t, ok := n.topicMap[topicName] n.RUnlock() if ok { return t } n.Lock() t, ok = n.topicMap[topicName] if ok { n.Unlock() return t } deleteCallback := func(t *Topic) { n.DeleteExistingTopic(t.name) } t = NewTopic(topicName, n, deleteCallback) n.topicMap[topicName] = t n.Unlock() n.logf(LOG_INFO, "TOPIC(%s): created", t.name) // topic is created but messagePump not yet started // if this topic was created while loading metadata at startup don't do any further initialization // (topic will be "started" after loading completes) if atomic.LoadInt32(&n.isLoading) == 1 { return t } // if using lookupd, make a blocking call to get channels and immediately create them // to ensure that all channels receive published messages lookupdHTTPAddrs := n.lookupdHTTPAddrs() if len(lookupdHTTPAddrs) > 0 { channelNames, err := n.ci.GetLookupdTopicChannels(t.name, lookupdHTTPAddrs) if err != nil { n.logf(LOG_WARN, "failed to query nsqlookupd for channels to pre-create for topic %s - %s", t.name, err) } for _, channelName := range channelNames { if strings.HasSuffix(channelName, "#ephemeral") { continue // do not create ephemeral channel with no consumer client } t.GetChannel(channelName) } } else if len(n.getOpts().NSQLookupdTCPAddresses) > 0 { n.logf(LOG_ERROR, "no available nsqlookupd to query for channels to pre-create for topic %s", t.name) } // now that all channels are added, start topic messagePump t.Start() return t } // GetExistingTopic gets a topic only if it exists func (n *NSQD) GetExistingTopic(topicName string) (*Topic, error) { n.RLock() defer n.RUnlock() topic, ok := n.topicMap[topicName] if !ok { return nil, errors.New("topic does not exist") } return topic, nil } // DeleteExistingTopic removes a topic only if it exists func (n *NSQD) DeleteExistingTopic(topicName string) error { n.RLock() topic, ok := n.topicMap[topicName] if !ok { n.RUnlock() return errors.New("topic does not exist") } n.RUnlock() // delete empties all channels and the topic itself before closing // (so that we dont leave any messages around) // // we do this before removing the topic from map below (with no lock) // so that any incoming writes will error and not create a new topic // to enforce ordering topic.Delete() n.Lock() delete(n.topicMap, topicName) n.Unlock() return nil } func (n *NSQD) Notify(v interface{}, persist bool) { // since the in-memory metadata is incomplete, // should not persist metadata while loading it. // nsqd will call `PersistMetadata` it after loading loading := atomic.LoadInt32(&n.isLoading) == 1 n.waitGroup.Wrap(func() { // by selecting on exitChan we guarantee that // we do not block exit, see issue #123 select { case <-n.exitChan: case n.notifyChan <- v: if loading || !persist { return } n.Lock() err := n.PersistMetadata() if err != nil { n.logf(LOG_ERROR, "failed to persist metadata - %s", err) } n.Unlock() } }) } // channels returns a flat slice of all channels in all topics func (n *NSQD) channels() []*Channel { var channels []*Channel n.RLock() for _, t := range n.topicMap { t.RLock() for _, c := range t.channelMap { channels = append(channels, c) } t.RUnlock() } n.RUnlock() return channels } // resizePool adjusts the size of the pool of queueScanWorker goroutines // // 1 <= pool <= min(num * 0.25, QueueScanWorkerPoolMax) func (n *NSQD) resizePool(num int, workCh chan *Channel, responseCh chan bool, closeCh chan int) { idealPoolSize := int(float64(num) * 0.25) if idealPoolSize < 1 { idealPoolSize = 1 } else if idealPoolSize > n.getOpts().QueueScanWorkerPoolMax { idealPoolSize = n.getOpts().QueueScanWorkerPoolMax } for { if idealPoolSize == n.poolSize { break } else if idealPoolSize < n.poolSize { // contract closeCh <- 1 n.poolSize-- } else { // expand n.waitGroup.Wrap(func() { n.queueScanWorker(workCh, responseCh, closeCh) }) n.poolSize++ } } } // queueScanWorker receives work (in the form of a channel) from queueScanLoop // and processes the deferred and in-flight queues func (n *NSQD) queueScanWorker(workCh chan *Channel, responseCh chan bool, closeCh chan int) { for { select { case c := <-workCh: now := time.Now().UnixNano() dirty := false if c.processInFlightQueue(now) { dirty = true } if c.processDeferredQueue(now) { dirty = true } responseCh <- dirty case <-closeCh: return } } } // queueScanLoop runs in a single goroutine to process in-flight and deferred // priority queues. It manages a pool of queueScanWorker (configurable max of // QueueScanWorkerPoolMax (default: 4)) that process channels concurrently. // // It copies Redis's probabilistic expiration algorithm: it wakes up every // QueueScanInterval (default: 100ms) to select a random QueueScanSelectionCount // (default: 20) channels from a locally cached list (refreshed every // QueueScanRefreshInterval (default: 5s)). // // If either of the queues had work to do the channel is considered "dirty". // // If QueueScanDirtyPercent (default: 25%) of the selected channels were dirty, // the loop continues without sleep. func (n *NSQD) queueScanLoop() { workCh := make(chan *Channel, n.getOpts().QueueScanSelectionCount) responseCh := make(chan bool, n.getOpts().QueueScanSelectionCount) closeCh := make(chan int) workTicker := time.NewTicker(n.getOpts().QueueScanInterval) refreshTicker := time.NewTicker(n.getOpts().QueueScanRefreshInterval) channels := n.channels() n.resizePool(len(channels), workCh, responseCh, closeCh) for { select { case <-workTicker.C: if len(channels) == 0 { continue } case <-refreshTicker.C: channels = n.channels() n.resizePool(len(channels), workCh, responseCh, closeCh) continue case <-n.exitChan: goto exit } num := n.getOpts().QueueScanSelectionCount if num > len(channels) { num = len(channels) } loop: for _, i := range util.UniqRands(num, len(channels)) { workCh <- channels[i] } numDirty := 0 for i := 0; i < num; i++ { if <-responseCh { numDirty++ } } if float64(numDirty)/float64(num) > n.getOpts().QueueScanDirtyPercent { goto loop } } exit: n.logf(LOG_INFO, "QUEUESCAN: closing") close(closeCh) workTicker.Stop() refreshTicker.Stop() } func buildTLSConfig(opts *Options) (*tls.Config, error) { var tlsConfig *tls.Config if opts.TLSCert == "" && opts.TLSKey == "" { return nil, nil } tlsClientAuthPolicy := tls.VerifyClientCertIfGiven cert, err := tls.LoadX509KeyPair(opts.TLSCert, opts.TLSKey) if err != nil { return nil, err } switch opts.TLSClientAuthPolicy { case "require": tlsClientAuthPolicy = tls.RequireAnyClientCert case "require-verify": tlsClientAuthPolicy = tls.RequireAndVerifyClientCert default: tlsClientAuthPolicy = tls.NoClientCert } tlsConfig = &tls.Config{ Certificates: []tls.Certificate{cert}, ClientAuth: tlsClientAuthPolicy, MinVersion: opts.TLSMinVersion, } if opts.TLSRootCAFile != "" { tlsCertPool := x509.NewCertPool() caCertFile, err := os.ReadFile(opts.TLSRootCAFile) if err != nil { return nil, err } if !tlsCertPool.AppendCertsFromPEM(caCertFile) { return nil, errors.New("failed to append certificate to pool") } tlsConfig.ClientCAs = tlsCertPool } return tlsConfig, nil } func buildClientTLSConfig(opts *Options) (*tls.Config, error) { tlsConfig := &tls.Config{ MinVersion: opts.TLSMinVersion, } if opts.TLSRootCAFile != "" { tlsCertPool := x509.NewCertPool() caCertFile, err := os.ReadFile(opts.TLSRootCAFile) if err != nil { return nil, err } if !tlsCertPool.AppendCertsFromPEM(caCertFile) { return nil, errors.New("failed to append certificate to pool") } tlsConfig.RootCAs = tlsCertPool } return tlsConfig, nil } func (n *NSQD) IsAuthEnabled() bool { return len(n.getOpts().AuthHTTPAddresses) != 0 } // Context returns a context that will be canceled when nsqd initiates the shutdown func (n *NSQD) Context() context.Context { return n.ctx } ================================================ FILE: nsqd/nsqd_test.go ================================================ package nsqd import ( "encoding/json" "errors" "fmt" "io/fs" "net" "os" "strconv" "sync/atomic" "testing" "time" "github.com/nsqio/nsq/internal/http_api" "github.com/nsqio/nsq/internal/test" "github.com/nsqio/nsq/nsqlookupd" ) const ( ConnectTimeout = 2 * time.Second RequestTimeout = 5 * time.Second ) func getMetadata(n *NSQD) (*Metadata, error) { fn := newMetadataFile(n.getOpts()) data, err := os.ReadFile(fn) if err != nil { return nil, err } var m Metadata err = json.Unmarshal(data, &m) if err != nil { return nil, err } return &m, nil } func TestStartup(t *testing.T) { var msg *Message iterations := 300 doneExitChan := make(chan int) opts := NewOptions() opts.Logger = test.NewTestLogger(t) opts.MemQueueSize = 100 opts.MaxBytesPerFile = 10240 _, _, nsqd := mustStartNSQD(opts) defer os.RemoveAll(opts.DataPath) origDataPath := opts.DataPath topicName := "nsqd_test" + strconv.Itoa(int(time.Now().Unix())) exitChan := make(chan int) go func() { <-exitChan nsqd.Exit() doneExitChan <- 1 }() // verify nsqd metadata shows no topics err := nsqd.PersistMetadata() test.Nil(t, err) atomic.StoreInt32(&nsqd.isLoading, 1) nsqd.GetTopic(topicName) // will not persist if `flagLoading` m, err := getMetadata(nsqd) test.Nil(t, err) test.Equal(t, 0, len(m.Topics)) nsqd.DeleteExistingTopic(topicName) atomic.StoreInt32(&nsqd.isLoading, 0) body := make([]byte, 256) topic := nsqd.GetTopic(topicName) for i := 0; i < iterations; i++ { msg := NewMessage(topic.GenerateID(), body) topic.PutMessage(msg) } t.Logf("pulling from channel") channel1 := topic.GetChannel("ch1") t.Logf("read %d msgs", iterations/2) for i := 0; i < iterations/2; i++ { select { case msg = <-channel1.memoryMsgChan: case b := <-channel1.backend.ReadChan(): msg, _ = decodeMessage(b) } t.Logf("read message %d", i+1) test.Equal(t, body, msg.Body) } for { if channel1.Depth() == int64(iterations/2) { break } time.Sleep(50 * time.Millisecond) } // make sure metadata shows the topic m, err = getMetadata(nsqd) test.Nil(t, err) test.Equal(t, 1, len(m.Topics)) test.Equal(t, topicName, m.Topics[0].Name) exitChan <- 1 <-doneExitChan // start up a new nsqd w/ the same folder opts = NewOptions() opts.Logger = test.NewTestLogger(t) opts.MemQueueSize = 100 opts.MaxBytesPerFile = 10240 opts.DataPath = origDataPath _, _, nsqd = mustStartNSQD(opts) go func() { <-exitChan nsqd.Exit() doneExitChan <- 1 }() topic = nsqd.GetTopic(topicName) // should be empty; channel should have drained everything count := topic.Depth() test.Equal(t, int64(0), count) channel1 = topic.GetChannel("ch1") for { if channel1.Depth() == int64(iterations/2) { break } time.Sleep(50 * time.Millisecond) } // read the other half of the messages for i := 0; i < iterations/2; i++ { select { case msg = <-channel1.memoryMsgChan: case b := <-channel1.backend.ReadChan(): msg, _ = decodeMessage(b) } t.Logf("read message %d", i+1) test.Equal(t, body, msg.Body) } // verify we drained things test.Equal(t, 0, len(topic.memoryMsgChan)) test.Equal(t, int64(0), topic.backend.Depth()) exitChan <- 1 <-doneExitChan } func TestEphemeralTopicsAndChannels(t *testing.T) { // ephemeral topics/channels are lazily removed after the last channel/client is removed opts := NewOptions() opts.Logger = test.NewTestLogger(t) opts.MemQueueSize = 100 _, _, nsqd := mustStartNSQD(opts) defer os.RemoveAll(opts.DataPath) topicName := "ephemeral_topic" + strconv.Itoa(int(time.Now().Unix())) + "#ephemeral" doneExitChan := make(chan int) exitChan := make(chan int) go func() { <-exitChan nsqd.Exit() doneExitChan <- 1 }() body := []byte("an_ephemeral_message") topic := nsqd.GetTopic(topicName) ephemeralChannel := topic.GetChannel("ch1#ephemeral") client := newClientV2(0, nil, nsqd) err := ephemeralChannel.AddClient(client.ID, client) test.Equal(t, err, nil) msg := NewMessage(topic.GenerateID(), body) topic.PutMessage(msg) msg = <-ephemeralChannel.memoryMsgChan test.Equal(t, body, msg.Body) ephemeralChannel.RemoveClient(client.ID) time.Sleep(100 * time.Millisecond) topic.Lock() numChannels := len(topic.channelMap) topic.Unlock() test.Equal(t, 0, numChannels) nsqd.Lock() numTopics := len(nsqd.topicMap) nsqd.Unlock() test.Equal(t, 0, numTopics) exitChan <- 1 <-doneExitChan } func TestPauseMetadata(t *testing.T) { opts := NewOptions() opts.Logger = test.NewTestLogger(t) _, _, nsqd := mustStartNSQD(opts) defer os.RemoveAll(opts.DataPath) defer nsqd.Exit() // avoid concurrency issue of async PersistMetadata() calls atomic.StoreInt32(&nsqd.isLoading, 1) topicName := "pause_metadata" + strconv.Itoa(int(time.Now().Unix())) topic := nsqd.GetTopic(topicName) channel := topic.GetChannel("ch") atomic.StoreInt32(&nsqd.isLoading, 0) nsqd.PersistMetadata() var isPaused = func(n *NSQD, topicIndex int, channelIndex int) bool { m, _ := getMetadata(n) return m.Topics[topicIndex].Channels[channelIndex].Paused } test.Equal(t, false, isPaused(nsqd, 0, 0)) channel.Pause() test.Equal(t, false, isPaused(nsqd, 0, 0)) nsqd.PersistMetadata() test.Equal(t, true, isPaused(nsqd, 0, 0)) channel.UnPause() test.Equal(t, true, isPaused(nsqd, 0, 0)) nsqd.PersistMetadata() test.Equal(t, false, isPaused(nsqd, 0, 0)) } func mustStartNSQLookupd(opts *nsqlookupd.Options) (net.Addr, net.Addr, *nsqlookupd.NSQLookupd) { opts.TCPAddress = "127.0.0.1:0" opts.HTTPAddress = "127.0.0.1:0" lookupd, err := nsqlookupd.New(opts) if err != nil { panic(err) } go func() { err := lookupd.Main() if err != nil { panic(err) } }() return lookupd.RealTCPAddr(), lookupd.RealHTTPAddr(), lookupd } func TestReconfigure(t *testing.T) { lopts := nsqlookupd.NewOptions() lopts.Logger = test.NewTestLogger(t) lopts1 := *lopts _, _, lookupd1 := mustStartNSQLookupd(&lopts1) defer lookupd1.Exit() lopts2 := *lopts _, _, lookupd2 := mustStartNSQLookupd(&lopts2) defer lookupd2.Exit() lopts3 := *lopts _, _, lookupd3 := mustStartNSQLookupd(&lopts3) defer lookupd3.Exit() opts := NewOptions() opts.Logger = test.NewTestLogger(t) _, _, nsqd := mustStartNSQD(opts) defer os.RemoveAll(opts.DataPath) defer nsqd.Exit() newOpts := NewOptions() newOpts.Logger = opts.Logger newOpts.NSQLookupdTCPAddresses = []string{lookupd1.RealTCPAddr().String()} nsqd.swapOpts(newOpts) nsqd.triggerOptsNotification() test.Equal(t, 1, len(nsqd.getOpts().NSQLookupdTCPAddresses)) var numLookupPeers int for i := 0; i < 100; i++ { numLookupPeers = len(nsqd.lookupPeers.Load().([]*lookupPeer)) if numLookupPeers == 1 { break } time.Sleep(10 * time.Millisecond) } test.Equal(t, 1, numLookupPeers) newOpts = NewOptions() newOpts.Logger = opts.Logger newOpts.NSQLookupdTCPAddresses = []string{lookupd2.RealTCPAddr().String(), lookupd3.RealTCPAddr().String()} nsqd.swapOpts(newOpts) nsqd.triggerOptsNotification() test.Equal(t, 2, len(nsqd.getOpts().NSQLookupdTCPAddresses)) for i := 0; i < 100; i++ { numLookupPeers = len(nsqd.lookupPeers.Load().([]*lookupPeer)) if numLookupPeers == 2 { break } time.Sleep(10 * time.Millisecond) } test.Equal(t, 2, numLookupPeers) var lookupPeers []string for _, lp := range nsqd.lookupPeers.Load().([]*lookupPeer) { lookupPeers = append(lookupPeers, lp.addr) } test.Equal(t, newOpts.NSQLookupdTCPAddresses, lookupPeers) } func TestCluster(t *testing.T) { lopts := nsqlookupd.NewOptions() lopts.Logger = test.NewTestLogger(t) lopts.BroadcastAddress = "127.0.0.1" _, _, lookupd := mustStartNSQLookupd(lopts) opts := NewOptions() opts.Logger = test.NewTestLogger(t) opts.NSQLookupdTCPAddresses = []string{lookupd.RealTCPAddr().String()} opts.BroadcastAddress = "127.0.0.1" _, _, nsqd := mustStartNSQD(opts) defer os.RemoveAll(opts.DataPath) defer nsqd.Exit() topicName := "cluster_test" + strconv.Itoa(int(time.Now().Unix())) hostname, err := os.Hostname() test.Nil(t, err) url := fmt.Sprintf("http://%s/topic/create?topic=%s", nsqd.RealHTTPAddr(), topicName) err = http_api.NewClient(nil, ConnectTimeout, RequestTimeout).POSTV1(url, nil, nil) test.Nil(t, err) url = fmt.Sprintf("http://%s/channel/create?topic=%s&channel=ch", nsqd.RealHTTPAddr(), topicName) err = http_api.NewClient(nil, ConnectTimeout, RequestTimeout).POSTV1(url, nil, nil) test.Nil(t, err) // allow some time for nsqd to push info to nsqlookupd time.Sleep(350 * time.Millisecond) var d map[string][]struct { Hostname string `json:"hostname"` BroadcastAddress string `json:"broadcast_address"` TCPPort int `json:"tcp_port"` Tombstoned bool `json:"tombstoned"` } endpoint := fmt.Sprintf("http://%s/debug", lookupd.RealHTTPAddr()) err = http_api.NewClient(nil, ConnectTimeout, RequestTimeout).GETV1(endpoint, &d) test.Nil(t, err) topicData := d["topic:"+topicName+":"] test.Equal(t, 1, len(topicData)) test.Equal(t, hostname, topicData[0].Hostname) test.Equal(t, "127.0.0.1", topicData[0].BroadcastAddress) test.Equal(t, nsqd.RealTCPAddr().(*net.TCPAddr).Port, topicData[0].TCPPort) test.Equal(t, false, topicData[0].Tombstoned) channelData := d["channel:"+topicName+":ch"] test.Equal(t, 1, len(channelData)) test.Equal(t, hostname, channelData[0].Hostname) test.Equal(t, "127.0.0.1", channelData[0].BroadcastAddress) test.Equal(t, nsqd.RealTCPAddr().(*net.TCPAddr).Port, channelData[0].TCPPort) test.Equal(t, false, channelData[0].Tombstoned) var lr struct { Producers []struct { Hostname string `json:"hostname"` BroadcastAddress string `json:"broadcast_address"` TCPPort int `json:"tcp_port"` } `json:"producers"` Channels []string `json:"channels"` } endpoint = fmt.Sprintf("http://%s/lookup?topic=%s", lookupd.RealHTTPAddr(), topicName) err = http_api.NewClient(nil, ConnectTimeout, RequestTimeout).GETV1(endpoint, &lr) test.Nil(t, err) test.Equal(t, 1, len(lr.Producers)) test.Equal(t, hostname, lr.Producers[0].Hostname) test.Equal(t, "127.0.0.1", lr.Producers[0].BroadcastAddress) test.Equal(t, nsqd.RealTCPAddr().(*net.TCPAddr).Port, lr.Producers[0].TCPPort) test.Equal(t, 1, len(lr.Channels)) test.Equal(t, "ch", lr.Channels[0]) url = fmt.Sprintf("http://%s/topic/delete?topic=%s", nsqd.RealHTTPAddr(), topicName) err = http_api.NewClient(nil, ConnectTimeout, RequestTimeout).POSTV1(url, nil, nil) test.Nil(t, err) // allow some time for nsqd to push info to nsqlookupd time.Sleep(350 * time.Millisecond) endpoint = fmt.Sprintf("http://%s/lookup?topic=%s", lookupd.RealHTTPAddr(), topicName) err = http_api.NewClient(nil, ConnectTimeout, RequestTimeout).GETV1(endpoint, &lr) test.Nil(t, err) test.Equal(t, 0, len(lr.Producers)) var dd map[string][]interface{} endpoint = fmt.Sprintf("http://%s/debug", lookupd.RealHTTPAddr()) err = http_api.NewClient(nil, ConnectTimeout, RequestTimeout).GETV1(endpoint, &dd) test.Nil(t, err) test.Equal(t, 0, len(dd["topic:"+topicName+":"])) test.Equal(t, 0, len(dd["channel:"+topicName+":ch"])) } func TestSetHealth(t *testing.T) { opts := NewOptions() opts.Logger = test.NewTestLogger(t) nsqd, err := New(opts) test.Nil(t, err) defer nsqd.Exit() test.Nil(t, nsqd.GetError()) test.Equal(t, true, nsqd.IsHealthy()) nsqd.SetHealth(nil) test.Nil(t, nsqd.GetError()) test.Equal(t, true, nsqd.IsHealthy()) nsqd.SetHealth(errors.New("health error")) test.NotNil(t, nsqd.GetError()) test.Equal(t, "NOK - health error", nsqd.GetHealth()) test.Equal(t, false, nsqd.IsHealthy()) nsqd.SetHealth(nil) test.Nil(t, nsqd.GetError()) test.Equal(t, "OK", nsqd.GetHealth()) test.Equal(t, true, nsqd.IsHealthy()) } func TestUnixSocketStartup(t *testing.T) { isSocket := func(path string) bool { fileInfo, err := os.Stat(path) if err != nil { return false } return fileInfo.Mode().Type() == fs.ModeSocket } opts := NewOptions() opts.Logger = test.NewTestLogger(t) _, _, nsqd := mustUnixSocketStartNSQD(opts) defer os.RemoveAll(opts.DataPath) defer nsqd.Exit() test.Equal(t, isSocket(opts.TCPAddress), true) test.Equal(t, isSocket(opts.HTTPAddress), true) } ================================================ FILE: nsqd/options.go ================================================ package nsqd import ( "crypto/md5" "crypto/tls" "hash/crc32" "io" "log" "os" "time" "github.com/nsqio/nsq/internal/lg" ) type Options struct { // basic options ID int64 `flag:"node-id" cfg:"id"` LogLevel lg.LogLevel `flag:"log-level"` LogPrefix string `flag:"log-prefix"` Logger Logger TCPAddress string `flag:"tcp-address"` HTTPAddress string `flag:"http-address"` HTTPSAddress string `flag:"https-address"` BroadcastAddress string `flag:"broadcast-address"` BroadcastTCPPort int `flag:"broadcast-tcp-port"` BroadcastHTTPPort int `flag:"broadcast-http-port"` NSQLookupdTCPAddresses []string `flag:"lookupd-tcp-address" cfg:"nsqlookupd_tcp_addresses"` AuthHTTPAddresses []string `flag:"auth-http-address" cfg:"auth_http_addresses"` AuthHTTPRequestMethod string `flag:"auth-http-request-method" cfg:"auth_http_request_method"` HTTPClientConnectTimeout time.Duration `flag:"http-client-connect-timeout" cfg:"http_client_connect_timeout"` HTTPClientRequestTimeout time.Duration `flag:"http-client-request-timeout" cfg:"http_client_request_timeout"` TopologyRegion string `flag:"topology-region"` TopologyZone string `flag:"topology-zone"` // diskqueue options DataPath string `flag:"data-path"` MemQueueSize int64 `flag:"mem-queue-size"` MaxBytesPerFile int64 `flag:"max-bytes-per-file"` SyncEvery int64 `flag:"sync-every"` SyncTimeout time.Duration `flag:"sync-timeout"` QueueScanInterval time.Duration QueueScanRefreshInterval time.Duration QueueScanSelectionCount int `flag:"queue-scan-selection-count"` QueueScanWorkerPoolMax int `flag:"queue-scan-worker-pool-max"` QueueScanDirtyPercent float64 // msg and command options MsgTimeout time.Duration `flag:"msg-timeout"` MaxMsgTimeout time.Duration `flag:"max-msg-timeout"` MaxMsgSize int64 `flag:"max-msg-size"` MaxBodySize int64 `flag:"max-body-size"` MaxReqTimeout time.Duration `flag:"max-req-timeout"` ClientTimeout time.Duration MaxDeferTimeout time.Duration `flag:"max-defer-timeout"` // client overridable configuration options MaxHeartbeatInterval time.Duration `flag:"max-heartbeat-interval"` MaxRdyCount int64 `flag:"max-rdy-count"` MaxOutputBufferSize int64 `flag:"max-output-buffer-size"` MaxOutputBufferTimeout time.Duration `flag:"max-output-buffer-timeout"` MinOutputBufferTimeout time.Duration `flag:"min-output-buffer-timeout"` OutputBufferTimeout time.Duration `flag:"output-buffer-timeout"` MaxChannelConsumers int `flag:"max-channel-consumers"` // statsd integration StatsdAddress string `flag:"statsd-address"` StatsdPrefix string `flag:"statsd-prefix"` StatsdInterval time.Duration `flag:"statsd-interval"` StatsdMemStats bool `flag:"statsd-mem-stats"` StatsdUDPPacketSize int `flag:"statsd-udp-packet-size"` StatsdExcludeEphemeral bool `flag:"statsd-exclude-ephemeral"` // e2e message latency E2EProcessingLatencyWindowTime time.Duration `flag:"e2e-processing-latency-window-time"` E2EProcessingLatencyPercentiles []float64 `flag:"e2e-processing-latency-percentile" cfg:"e2e_processing_latency_percentiles"` // TLS config TLSCert string `flag:"tls-cert"` TLSKey string `flag:"tls-key"` TLSClientAuthPolicy string `flag:"tls-client-auth-policy"` TLSRootCAFile string `flag:"tls-root-ca-file"` TLSRequired int `flag:"tls-required"` TLSMinVersion uint16 `flag:"tls-min-version"` // compression DeflateEnabled bool `flag:"deflate"` MaxDeflateLevel int `flag:"max-deflate-level"` SnappyEnabled bool `flag:"snappy"` // experimental features Experiments []string `flag:"enable-experiment" cfg:"enable_experiment"` } type Experiment string const ( TopologyAwareConsumption Experiment = "topology-aware-consumption" ) var AllExperiments = []Experiment{ TopologyAwareConsumption, } func (o Options) HasExperiment(e Experiment) bool { for _, v := range o.Experiments { if string(e) == v { return true } } return false } func NewOptions() *Options { hostname, err := os.Hostname() if err != nil { log.Fatal(err) } h := md5.New() io.WriteString(h, hostname) defaultID := int64(crc32.ChecksumIEEE(h.Sum(nil)) % 1024) return &Options{ ID: defaultID, LogPrefix: "[nsqd] ", LogLevel: lg.INFO, TCPAddress: "0.0.0.0:4150", HTTPAddress: "0.0.0.0:4151", HTTPSAddress: "0.0.0.0:4152", BroadcastAddress: hostname, BroadcastTCPPort: 0, BroadcastHTTPPort: 0, NSQLookupdTCPAddresses: make([]string, 0), AuthHTTPAddresses: make([]string, 0), AuthHTTPRequestMethod: "get", HTTPClientConnectTimeout: 2 * time.Second, HTTPClientRequestTimeout: 5 * time.Second, MemQueueSize: 10000, MaxBytesPerFile: 100 * 1024 * 1024, SyncEvery: 2500, SyncTimeout: 2 * time.Second, QueueScanInterval: 100 * time.Millisecond, QueueScanRefreshInterval: 5 * time.Second, QueueScanSelectionCount: 20, QueueScanWorkerPoolMax: 4, QueueScanDirtyPercent: 0.25, MsgTimeout: 60 * time.Second, MaxMsgTimeout: 15 * time.Minute, MaxMsgSize: 1024 * 1024, MaxBodySize: 5 * 1024 * 1024, MaxReqTimeout: 1 * time.Hour, ClientTimeout: 60 * time.Second, MaxDeferTimeout: 1 * time.Hour, MaxHeartbeatInterval: 60 * time.Second, MaxRdyCount: 2500, MaxOutputBufferSize: 64 * 1024, MaxOutputBufferTimeout: 30 * time.Second, MinOutputBufferTimeout: 25 * time.Millisecond, OutputBufferTimeout: 250 * time.Millisecond, MaxChannelConsumers: 0, StatsdPrefix: "nsq.%s", StatsdInterval: 60 * time.Second, StatsdMemStats: true, StatsdUDPPacketSize: 508, E2EProcessingLatencyWindowTime: time.Duration(10 * time.Minute), DeflateEnabled: true, MaxDeflateLevel: 6, SnappyEnabled: true, TLSMinVersion: tls.VersionTLS10, } } ================================================ FILE: nsqd/protocol_v2.go ================================================ package nsqd import ( "bytes" "encoding/binary" "encoding/json" "errors" "fmt" "io" "math/rand" "net" "sync/atomic" "time" "unsafe" "github.com/nsqio/nsq/internal/protocol" "github.com/nsqio/nsq/internal/version" ) const ( frameTypeResponse int32 = 0 frameTypeError int32 = 1 frameTypeMessage int32 = 2 ) var separatorBytes = []byte(" ") var heartbeatBytes = []byte("_heartbeat_") var okBytes = []byte("OK") type protocolV2 struct { nsqd *NSQD } func (p *protocolV2) NewClient(conn net.Conn) protocol.Client { clientID := atomic.AddInt64(&p.nsqd.clientIDSequence, 1) return newClientV2(clientID, conn, p.nsqd) } func (p *protocolV2) IOLoop(c protocol.Client) error { var err error var line []byte var zeroTime time.Time client := c.(*clientV2) // synchronize the startup of messagePump in order // to guarantee that it gets a chance to initialize // goroutine local state derived from client attributes // and avoid a potential race with IDENTIFY (where a client // could have changed or disabled said attributes) messagePumpStartedChan := make(chan bool) go p.messagePump(client, messagePumpStartedChan) <-messagePumpStartedChan for { if client.HeartbeatInterval > 0 { client.SetReadDeadline(time.Now().Add(client.HeartbeatInterval * 2)) } else { client.SetReadDeadline(zeroTime) } // ReadSlice does not allocate new space for the data each request // ie. the returned slice is only valid until the next call to it line, err = client.Reader.ReadSlice('\n') if err != nil { if err == io.EOF { err = nil } else { err = fmt.Errorf("failed to read command - %s", err) } break } // trim the '\n' line = line[:len(line)-1] // optionally trim the '\r' if len(line) > 0 && line[len(line)-1] == '\r' { line = line[:len(line)-1] } params := bytes.Split(line, separatorBytes) p.nsqd.logf(LOG_DEBUG, "PROTOCOL(V2): [%s] %s", client, params) var response []byte response, err = p.Exec(client, params) if err != nil { ctx := "" if parentErr := err.(protocol.ChildErr).Parent(); parentErr != nil { ctx = " - " + parentErr.Error() } p.nsqd.logf(LOG_ERROR, "[%s] - %s%s", client, err, ctx) sendErr := p.Send(client, frameTypeError, []byte(err.Error())) if sendErr != nil { p.nsqd.logf(LOG_ERROR, "[%s] - %s%s", client, sendErr, ctx) break } // errors of type FatalClientErr should forceably close the connection if _, ok := err.(*protocol.FatalClientErr); ok { break } continue } if response != nil { err = p.Send(client, frameTypeResponse, response) if err != nil { err = fmt.Errorf("failed to send response - %s", err) break } } } p.nsqd.logf(LOG_INFO, "PROTOCOL(V2): [%s] exiting ioloop", client) close(client.ExitChan) if client.Channel != nil { client.Channel.RemoveClient(client.ID) } return err } func (p *protocolV2) SendMessage(client *clientV2, msg *Message) error { p.nsqd.logf(LOG_DEBUG, "PROTOCOL(V2): writing msg(%s) to client(%s) - %s", msg.ID, client, msg.Body) buf := bufferPoolGet() defer bufferPoolPut(buf) _, err := msg.WriteTo(buf) if err != nil { return err } err = p.Send(client, frameTypeMessage, buf.Bytes()) if err != nil { return err } return nil } func (p *protocolV2) Send(client *clientV2, frameType int32, data []byte) error { client.writeLock.Lock() var zeroTime time.Time if client.HeartbeatInterval > 0 { client.SetWriteDeadline(time.Now().Add(client.HeartbeatInterval)) } else { client.SetWriteDeadline(zeroTime) } _, err := protocol.SendFramedResponse(client.Writer, frameType, data) if err != nil { client.writeLock.Unlock() return err } if frameType != frameTypeMessage { err = client.Flush() } client.writeLock.Unlock() return err } func (p *protocolV2) Exec(client *clientV2, params [][]byte) ([]byte, error) { if bytes.Equal(params[0], []byte("IDENTIFY")) { return p.IDENTIFY(client, params) } err := enforceTLSPolicy(client, p, params[0]) if err != nil { return nil, err } switch { case bytes.Equal(params[0], []byte("FIN")): return p.FIN(client, params) case bytes.Equal(params[0], []byte("RDY")): return p.RDY(client, params) case bytes.Equal(params[0], []byte("REQ")): return p.REQ(client, params) case bytes.Equal(params[0], []byte("PUB")): return p.PUB(client, params) case bytes.Equal(params[0], []byte("MPUB")): return p.MPUB(client, params) case bytes.Equal(params[0], []byte("DPUB")): return p.DPUB(client, params) case bytes.Equal(params[0], []byte("NOP")): return p.NOP(client, params) case bytes.Equal(params[0], []byte("TOUCH")): return p.TOUCH(client, params) case bytes.Equal(params[0], []byte("SUB")): return p.SUB(client, params) case bytes.Equal(params[0], []byte("CLS")): return p.CLS(client, params) case bytes.Equal(params[0], []byte("AUTH")): return p.AUTH(client, params) } return nil, protocol.NewFatalClientErr(nil, "E_INVALID", fmt.Sprintf("invalid command %s", params[0])) } func (p *protocolV2) messagePump(client *clientV2, startedChan chan bool) { var err error var zoneMsgChan, regionMsgChan, memoryMsgChan chan *Message var backendMsgChan <-chan []byte var subChannel *Channel // NOTE: `flusherChan` is used to bound message latency for // the pathological case of a channel on a low volume topic // with >1 clients having >1 RDY counts var flusherChan <-chan time.Time var sampleRate int32 var regionLocal, zoneLocal bool subEventChan := client.SubEventChan identifyEventChan := client.IdentifyEventChan outputBufferTicker := time.NewTicker(client.OutputBufferTimeout) heartbeatTicker := time.NewTicker(client.HeartbeatInterval) heartbeatChan := heartbeatTicker.C msgTimeout := client.MsgTimeout // v2 opportunistically buffers data to clients to reduce write system calls // we force flush in two cases: // 1. when the client is not ready to receive messages // 2. we're buffered and the channel has nothing left to send us // (ie. we would block in this loop anyway) // flushed := true // signal to the goroutine that started the messagePump // that we've started up close(startedChan) for { var b []byte var msg *Message if subChannel == nil || !client.IsReadyForMessages() { // the client is not ready to receive messages... memoryMsgChan = nil regionMsgChan = nil zoneMsgChan = nil backendMsgChan = nil flusherChan = nil // force flush client.writeLock.Lock() err = client.Flush() client.writeLock.Unlock() if err != nil { goto exit } flushed = true } else if flushed { // last iteration we flushed... // do not select on the flusher ticker channel memoryMsgChan = subChannel.memoryMsgChan if zoneLocal { zoneMsgChan = subChannel.zoneLocalMsgChan } if regionLocal { regionMsgChan = subChannel.regionLocalMsgChan } backendMsgChan = subChannel.backend.ReadChan() flusherChan = nil } else { // we're buffered (if there isn't any more data we should flush)... // select on the flusher ticker channel, too memoryMsgChan = subChannel.memoryMsgChan if zoneLocal { zoneMsgChan = subChannel.zoneLocalMsgChan } if regionLocal { regionMsgChan = subChannel.regionLocalMsgChan } backendMsgChan = subChannel.backend.ReadChan() flusherChan = outputBufferTicker.C } select { case <-flusherChan: // if this case wins, we're either starved // or we won the race between other channels... // in either case, force flush client.writeLock.Lock() err = client.Flush() client.writeLock.Unlock() if err != nil { goto exit } flushed = true case <-client.ReadyStateChan: case subChannel = <-subEventChan: // you can't SUB anymore subEventChan = nil case identifyData := <-identifyEventChan: // you can't IDENTIFY anymore identifyEventChan = nil outputBufferTicker.Stop() if identifyData.OutputBufferTimeout > 0 { outputBufferTicker = time.NewTicker(identifyData.OutputBufferTimeout) } heartbeatTicker.Stop() heartbeatChan = nil if identifyData.HeartbeatInterval > 0 { heartbeatTicker = time.NewTicker(identifyData.HeartbeatInterval) heartbeatChan = heartbeatTicker.C } if identifyData.SampleRate > 0 { sampleRate = identifyData.SampleRate } msgTimeout = identifyData.MsgTimeout isToplogyAware := p.nsqd.getOpts().HasExperiment(TopologyAwareConsumption) if identifyData.TopologyZone == p.nsqd.getOpts().TopologyZone && isToplogyAware { zoneLocal = true } if identifyData.TopologyRegion == p.nsqd.getOpts().TopologyRegion && isToplogyAware { regionLocal = true } case <-heartbeatChan: err = p.Send(client, frameTypeResponse, heartbeatBytes) if err != nil { goto exit } case b = <-backendMsgChan: // decodeMessage then handle 'msg' case msg = <-zoneMsgChan: atomic.AddUint64(&client.Channel.zoneLocalMsgCount, 1) case msg = <-regionMsgChan: if zoneLocal { atomic.AddUint64(&client.Channel.zoneLocalMsgCount, 1) } else { atomic.AddUint64(&client.Channel.regionLocalMsgCount, 1) } case msg = <-memoryMsgChan: if zoneLocal { atomic.AddUint64(&client.Channel.zoneLocalMsgCount, 1) } else if regionLocal { atomic.AddUint64(&client.Channel.regionLocalMsgCount, 1) } else { atomic.AddUint64(&client.Channel.globalMsgCount, 1) } case <-client.ExitChan: goto exit } if len(b) != 0 { msg, err = decodeMessage(b) if err != nil { p.nsqd.logf(LOG_ERROR, "failed to decode message - %s", err) continue } } if msg != nil { if sampleRate > 0 && rand.Int31n(100) > sampleRate { continue } msg.Attempts++ subChannel.StartInFlightTimeout(msg, client.ID, msgTimeout) client.SendingMessage() err = p.SendMessage(client, msg) if err != nil { goto exit } flushed = false } } exit: p.nsqd.logf(LOG_INFO, "PROTOCOL(V2): [%s] exiting messagePump", client) heartbeatTicker.Stop() outputBufferTicker.Stop() if err != nil { p.nsqd.logf(LOG_ERROR, "PROTOCOL(V2): [%s] messagePump error - %s", client, err) } } func (p *protocolV2) IDENTIFY(client *clientV2, params [][]byte) ([]byte, error) { var err error if atomic.LoadInt32(&client.State) != stateInit { return nil, protocol.NewFatalClientErr(nil, "E_INVALID", "cannot IDENTIFY in current state") } bodyLen, err := readLen(client.Reader, client.lenSlice) if err != nil { return nil, protocol.NewFatalClientErr(err, "E_BAD_BODY", "IDENTIFY failed to read body size") } if int64(bodyLen) > p.nsqd.getOpts().MaxBodySize { return nil, protocol.NewFatalClientErr(nil, "E_BAD_BODY", fmt.Sprintf("IDENTIFY body too big %d > %d", bodyLen, p.nsqd.getOpts().MaxBodySize)) } if bodyLen <= 0 { return nil, protocol.NewFatalClientErr(nil, "E_BAD_BODY", fmt.Sprintf("IDENTIFY invalid body size %d", bodyLen)) } body := make([]byte, bodyLen) _, err = io.ReadFull(client.Reader, body) if err != nil { return nil, protocol.NewFatalClientErr(err, "E_BAD_BODY", "IDENTIFY failed to read body") } // body is a json structure with producer information var identifyData identifyDataV2 err = json.Unmarshal(body, &identifyData) if err != nil { return nil, protocol.NewFatalClientErr(err, "E_BAD_BODY", "IDENTIFY failed to decode JSON body") } p.nsqd.logf(LOG_DEBUG, "PROTOCOL(V2): [%s] %+v", client, identifyData) err = client.Identify(identifyData) if err != nil { return nil, protocol.NewFatalClientErr(err, "E_BAD_BODY", "IDENTIFY "+err.Error()) } // bail out early if we're not negotiating features if !identifyData.FeatureNegotiation { return okBytes, nil } tlsv1 := p.nsqd.tlsConfig != nil && identifyData.TLSv1 deflate := p.nsqd.getOpts().DeflateEnabled && identifyData.Deflate deflateLevel := 6 if deflate && identifyData.DeflateLevel > 0 { deflateLevel = identifyData.DeflateLevel } if max := p.nsqd.getOpts().MaxDeflateLevel; max < deflateLevel { deflateLevel = max } snappy := p.nsqd.getOpts().SnappyEnabled && identifyData.Snappy if deflate && snappy { return nil, protocol.NewFatalClientErr(nil, "E_IDENTIFY_FAILED", "cannot enable both deflate and snappy compression") } resp, err := json.Marshal(struct { MaxRdyCount int64 `json:"max_rdy_count"` Version string `json:"version"` MaxMsgTimeout int64 `json:"max_msg_timeout"` MsgTimeout int64 `json:"msg_timeout"` TLSv1 bool `json:"tls_v1"` Deflate bool `json:"deflate"` DeflateLevel int `json:"deflate_level"` MaxDeflateLevel int `json:"max_deflate_level"` Snappy bool `json:"snappy"` SampleRate int32 `json:"sample_rate"` AuthRequired bool `json:"auth_required"` OutputBufferSize int `json:"output_buffer_size"` OutputBufferTimeout int64 `json:"output_buffer_timeout"` TopologyRegion string `json:"topology_region"` TopologyZone string `json:"topology_zone"` }{ MaxRdyCount: p.nsqd.getOpts().MaxRdyCount, Version: version.Binary, MaxMsgTimeout: int64(p.nsqd.getOpts().MaxMsgTimeout / time.Millisecond), MsgTimeout: int64(client.MsgTimeout / time.Millisecond), TLSv1: tlsv1, Deflate: deflate, DeflateLevel: deflateLevel, MaxDeflateLevel: p.nsqd.getOpts().MaxDeflateLevel, Snappy: snappy, SampleRate: client.SampleRate, AuthRequired: p.nsqd.IsAuthEnabled(), OutputBufferSize: client.OutputBufferSize, OutputBufferTimeout: int64(client.OutputBufferTimeout / time.Millisecond), TopologyRegion: p.nsqd.getOpts().TopologyRegion, TopologyZone: p.nsqd.getOpts().TopologyZone, }) if err != nil { return nil, protocol.NewFatalClientErr(err, "E_IDENTIFY_FAILED", "IDENTIFY failed "+err.Error()) } err = p.Send(client, frameTypeResponse, resp) if err != nil { return nil, protocol.NewFatalClientErr(err, "E_IDENTIFY_FAILED", "IDENTIFY failed "+err.Error()) } if tlsv1 { p.nsqd.logf(LOG_INFO, "PROTOCOL(V2): [%s] upgrading connection to TLS", client) err = client.UpgradeTLS() if err != nil { return nil, protocol.NewFatalClientErr(err, "E_IDENTIFY_FAILED", "IDENTIFY failed "+err.Error()) } err = p.Send(client, frameTypeResponse, okBytes) if err != nil { return nil, protocol.NewFatalClientErr(err, "E_IDENTIFY_FAILED", "IDENTIFY failed "+err.Error()) } } if snappy { p.nsqd.logf(LOG_INFO, "PROTOCOL(V2): [%s] upgrading connection to snappy", client) err = client.UpgradeSnappy() if err != nil { return nil, protocol.NewFatalClientErr(err, "E_IDENTIFY_FAILED", "IDENTIFY failed "+err.Error()) } err = p.Send(client, frameTypeResponse, okBytes) if err != nil { return nil, protocol.NewFatalClientErr(err, "E_IDENTIFY_FAILED", "IDENTIFY failed "+err.Error()) } } if deflate { p.nsqd.logf(LOG_INFO, "PROTOCOL(V2): [%s] upgrading connection to deflate (level %d)", client, deflateLevel) err = client.UpgradeDeflate(deflateLevel) if err != nil { return nil, protocol.NewFatalClientErr(err, "E_IDENTIFY_FAILED", "IDENTIFY failed "+err.Error()) } err = p.Send(client, frameTypeResponse, okBytes) if err != nil { return nil, protocol.NewFatalClientErr(err, "E_IDENTIFY_FAILED", "IDENTIFY failed "+err.Error()) } } return nil, nil } func (p *protocolV2) AUTH(client *clientV2, params [][]byte) ([]byte, error) { if atomic.LoadInt32(&client.State) != stateInit { return nil, protocol.NewFatalClientErr(nil, "E_INVALID", "cannot AUTH in current state") } if len(params) != 1 { return nil, protocol.NewFatalClientErr(nil, "E_INVALID", "AUTH invalid number of parameters") } bodyLen, err := readLen(client.Reader, client.lenSlice) if err != nil { return nil, protocol.NewFatalClientErr(err, "E_BAD_BODY", "AUTH failed to read body size") } if int64(bodyLen) > p.nsqd.getOpts().MaxBodySize { return nil, protocol.NewFatalClientErr(nil, "E_BAD_BODY", fmt.Sprintf("AUTH body too big %d > %d", bodyLen, p.nsqd.getOpts().MaxBodySize)) } if bodyLen <= 0 { return nil, protocol.NewFatalClientErr(nil, "E_BAD_BODY", fmt.Sprintf("AUTH invalid body size %d", bodyLen)) } body := make([]byte, bodyLen) _, err = io.ReadFull(client.Reader, body) if err != nil { return nil, protocol.NewFatalClientErr(err, "E_BAD_BODY", "AUTH failed to read body") } if client.HasAuthorizations() { return nil, protocol.NewFatalClientErr(nil, "E_INVALID", "AUTH already set") } if !client.nsqd.IsAuthEnabled() { return nil, protocol.NewFatalClientErr(err, "E_AUTH_DISABLED", "AUTH disabled") } if err := client.Auth(string(body)); err != nil { // we don't want to leak errors contacting the auth server to untrusted clients p.nsqd.logf(LOG_WARN, "PROTOCOL(V2): [%s] AUTH failed %s", client, err) return nil, protocol.NewFatalClientErr(err, "E_AUTH_FAILED", "AUTH failed") } if !client.HasAuthorizations() { return nil, protocol.NewFatalClientErr(nil, "E_UNAUTHORIZED", "AUTH no authorizations found") } resp, err := json.Marshal(struct { Identity string `json:"identity"` IdentityURL string `json:"identity_url"` PermissionCount int `json:"permission_count"` }{ Identity: client.AuthState.Identity, IdentityURL: client.AuthState.IdentityURL, PermissionCount: len(client.AuthState.Authorizations), }) if err != nil { return nil, protocol.NewFatalClientErr(err, "E_AUTH_ERROR", "AUTH error "+err.Error()) } err = p.Send(client, frameTypeResponse, resp) if err != nil { return nil, protocol.NewFatalClientErr(err, "E_AUTH_ERROR", "AUTH error "+err.Error()) } return nil, nil } func (p *protocolV2) CheckAuth(client *clientV2, cmd, topicName, channelName string) error { // if auth is enabled, the client must have authorized already // compare topic/channel against cached authorization data (refetching if expired) if client.nsqd.IsAuthEnabled() { if !client.HasAuthorizations() { return protocol.NewFatalClientErr(nil, "E_AUTH_FIRST", fmt.Sprintf("AUTH required before %s", cmd)) } ok, err := client.IsAuthorized(topicName, channelName) if err != nil { // we don't want to leak errors contacting the auth server to untrusted clients p.nsqd.logf(LOG_WARN, "PROTOCOL(V2): [%s] AUTH failed %s", client, err) return protocol.NewFatalClientErr(nil, "E_AUTH_FAILED", "AUTH failed") } if !ok { return protocol.NewFatalClientErr(nil, "E_UNAUTHORIZED", fmt.Sprintf("AUTH failed for %s on %q %q", cmd, topicName, channelName)) } } return nil } func (p *protocolV2) SUB(client *clientV2, params [][]byte) ([]byte, error) { if atomic.LoadInt32(&client.State) != stateInit { return nil, protocol.NewFatalClientErr(nil, "E_INVALID", "cannot SUB in current state") } if client.HeartbeatInterval <= 0 { return nil, protocol.NewFatalClientErr(nil, "E_INVALID", "cannot SUB with heartbeats disabled") } if len(params) < 3 { return nil, protocol.NewFatalClientErr(nil, "E_INVALID", "SUB insufficient number of parameters") } topicName := string(params[1]) if !protocol.IsValidTopicName(topicName) { return nil, protocol.NewFatalClientErr(nil, "E_BAD_TOPIC", fmt.Sprintf("SUB topic name %q is not valid", topicName)) } channelName := string(params[2]) if !protocol.IsValidChannelName(channelName) { return nil, protocol.NewFatalClientErr(nil, "E_BAD_CHANNEL", fmt.Sprintf("SUB channel name %q is not valid", channelName)) } if err := p.CheckAuth(client, "SUB", topicName, channelName); err != nil { return nil, err } // This retry-loop is a work-around for a race condition, where the // last client can leave the channel between GetChannel() and AddClient(). // Avoid adding a client to an ephemeral channel / topic which has started exiting. var channel *Channel for i := 1; ; i++ { topic := p.nsqd.GetTopic(topicName) channel = topic.GetChannel(channelName) if err := channel.AddClient(client.ID, client); err != nil { return nil, protocol.NewFatalClientErr(err, "E_SUB_FAILED", "SUB failed "+err.Error()) } if (channel.ephemeral && channel.Exiting()) || (topic.ephemeral && topic.Exiting()) { channel.RemoveClient(client.ID) if i < 2 { time.Sleep(100 * time.Millisecond) continue } return nil, protocol.NewFatalClientErr(nil, "E_SUB_FAILED", "SUB failed to deleted topic/channel") } break } atomic.StoreInt32(&client.State, stateSubscribed) client.Channel = channel // update message pump client.SubEventChan <- channel return okBytes, nil } func (p *protocolV2) RDY(client *clientV2, params [][]byte) ([]byte, error) { state := atomic.LoadInt32(&client.State) if state == stateClosing { // just ignore ready changes on a closing channel p.nsqd.logf(LOG_INFO, "PROTOCOL(V2): [%s] ignoring RDY after CLS in state ClientStateV2Closing", client) return nil, nil } if state != stateSubscribed { return nil, protocol.NewFatalClientErr(nil, "E_INVALID", "cannot RDY in current state") } count := int64(1) if len(params) > 1 { b10, err := protocol.ByteToBase10(params[1]) if err != nil { return nil, protocol.NewFatalClientErr(err, "E_INVALID", fmt.Sprintf("RDY could not parse count %s", params[1])) } count = int64(b10) } if count < 0 || count > p.nsqd.getOpts().MaxRdyCount { // this needs to be a fatal error otherwise clients would have // inconsistent state return nil, protocol.NewFatalClientErr(nil, "E_INVALID", fmt.Sprintf("RDY count %d out of range 0-%d", count, p.nsqd.getOpts().MaxRdyCount)) } client.SetReadyCount(count) return nil, nil } func (p *protocolV2) FIN(client *clientV2, params [][]byte) ([]byte, error) { state := atomic.LoadInt32(&client.State) if state != stateSubscribed && state != stateClosing { return nil, protocol.NewFatalClientErr(nil, "E_INVALID", "cannot FIN in current state") } if len(params) < 2 { return nil, protocol.NewFatalClientErr(nil, "E_INVALID", "FIN insufficient number of params") } id, err := getMessageID(params[1]) if err != nil { return nil, protocol.NewFatalClientErr(nil, "E_INVALID", err.Error()) } err = client.Channel.FinishMessage(client.ID, *id) if err != nil { return nil, protocol.NewClientErr(err, "E_FIN_FAILED", fmt.Sprintf("FIN %s failed %s", *id, err.Error())) } client.FinishedMessage() return nil, nil } func (p *protocolV2) REQ(client *clientV2, params [][]byte) ([]byte, error) { state := atomic.LoadInt32(&client.State) if state != stateSubscribed && state != stateClosing { return nil, protocol.NewFatalClientErr(nil, "E_INVALID", "cannot REQ in current state") } if len(params) < 3 { return nil, protocol.NewFatalClientErr(nil, "E_INVALID", "REQ insufficient number of params") } id, err := getMessageID(params[1]) if err != nil { return nil, protocol.NewFatalClientErr(nil, "E_INVALID", err.Error()) } timeoutMs, err := protocol.ByteToBase10(params[2]) if err != nil { return nil, protocol.NewFatalClientErr(err, "E_INVALID", fmt.Sprintf("REQ could not parse timeout %s", params[2])) } timeoutDuration := time.Duration(timeoutMs) * time.Millisecond maxReqTimeout := p.nsqd.getOpts().MaxReqTimeout clampedTimeout := timeoutDuration if timeoutDuration < 0 { clampedTimeout = 0 } else if timeoutDuration > maxReqTimeout { clampedTimeout = maxReqTimeout } if clampedTimeout != timeoutDuration { p.nsqd.logf(LOG_INFO, "PROTOCOL(V2): [%s] REQ timeout %d out of range 0-%d. Setting to %d", client, timeoutDuration, maxReqTimeout, clampedTimeout) timeoutDuration = clampedTimeout } err = client.Channel.RequeueMessage(client.ID, *id, timeoutDuration) if err != nil { return nil, protocol.NewClientErr(err, "E_REQ_FAILED", fmt.Sprintf("REQ %s failed %s", *id, err.Error())) } client.RequeuedMessage() return nil, nil } func (p *protocolV2) CLS(client *clientV2, params [][]byte) ([]byte, error) { if atomic.LoadInt32(&client.State) != stateSubscribed { return nil, protocol.NewFatalClientErr(nil, "E_INVALID", "cannot CLS in current state") } client.StartClose() return []byte("CLOSE_WAIT"), nil } func (p *protocolV2) NOP(client *clientV2, params [][]byte) ([]byte, error) { return nil, nil } func (p *protocolV2) PUB(client *clientV2, params [][]byte) ([]byte, error) { var err error if len(params) < 2 { return nil, protocol.NewFatalClientErr(nil, "E_INVALID", "PUB insufficient number of parameters") } topicName := string(params[1]) if !protocol.IsValidTopicName(topicName) { return nil, protocol.NewFatalClientErr(nil, "E_BAD_TOPIC", fmt.Sprintf("PUB topic name %q is not valid", topicName)) } bodyLen, err := readLen(client.Reader, client.lenSlice) if err != nil { return nil, protocol.NewFatalClientErr(err, "E_BAD_MESSAGE", "PUB failed to read message body size") } if bodyLen <= 0 { return nil, protocol.NewFatalClientErr(nil, "E_BAD_MESSAGE", fmt.Sprintf("PUB invalid message body size %d", bodyLen)) } if int64(bodyLen) > p.nsqd.getOpts().MaxMsgSize { return nil, protocol.NewFatalClientErr(nil, "E_BAD_MESSAGE", fmt.Sprintf("PUB message too big %d > %d", bodyLen, p.nsqd.getOpts().MaxMsgSize)) } messageBody := make([]byte, bodyLen) _, err = io.ReadFull(client.Reader, messageBody) if err != nil { return nil, protocol.NewFatalClientErr(err, "E_BAD_MESSAGE", "PUB failed to read message body") } if err := p.CheckAuth(client, "PUB", topicName, ""); err != nil { return nil, err } topic := p.nsqd.GetTopic(topicName) msg := NewMessage(topic.GenerateID(), messageBody) err = topic.PutMessage(msg) if err != nil { return nil, protocol.NewFatalClientErr(err, "E_PUB_FAILED", "PUB failed "+err.Error()) } client.PublishedMessage(topicName, 1) return okBytes, nil } func (p *protocolV2) MPUB(client *clientV2, params [][]byte) ([]byte, error) { var err error if len(params) < 2 { return nil, protocol.NewFatalClientErr(nil, "E_INVALID", "MPUB insufficient number of parameters") } topicName := string(params[1]) if !protocol.IsValidTopicName(topicName) { return nil, protocol.NewFatalClientErr(nil, "E_BAD_TOPIC", fmt.Sprintf("E_BAD_TOPIC MPUB topic name %q is not valid", topicName)) } if err := p.CheckAuth(client, "MPUB", topicName, ""); err != nil { return nil, err } topic := p.nsqd.GetTopic(topicName) bodyLen, err := readLen(client.Reader, client.lenSlice) if err != nil { return nil, protocol.NewFatalClientErr(err, "E_BAD_BODY", "MPUB failed to read body size") } if bodyLen <= 0 { return nil, protocol.NewFatalClientErr(nil, "E_BAD_BODY", fmt.Sprintf("MPUB invalid body size %d", bodyLen)) } if int64(bodyLen) > p.nsqd.getOpts().MaxBodySize { return nil, protocol.NewFatalClientErr(nil, "E_BAD_BODY", fmt.Sprintf("MPUB body too big %d > %d", bodyLen, p.nsqd.getOpts().MaxBodySize)) } messages, err := readMPUB(client.Reader, client.lenSlice, topic, p.nsqd.getOpts().MaxMsgSize, p.nsqd.getOpts().MaxBodySize) if err != nil { return nil, err } // if we've made it this far we've validated all the input, // the only possible error is that the topic is exiting during // this next call (and no messages will be queued in that case) err = topic.PutMessages(messages) if err != nil { return nil, protocol.NewFatalClientErr(err, "E_MPUB_FAILED", "MPUB failed "+err.Error()) } client.PublishedMessage(topicName, uint64(len(messages))) return okBytes, nil } func (p *protocolV2) DPUB(client *clientV2, params [][]byte) ([]byte, error) { var err error if len(params) < 3 { return nil, protocol.NewFatalClientErr(nil, "E_INVALID", "DPUB insufficient number of parameters") } topicName := string(params[1]) if !protocol.IsValidTopicName(topicName) { return nil, protocol.NewFatalClientErr(nil, "E_BAD_TOPIC", fmt.Sprintf("DPUB topic name %q is not valid", topicName)) } timeoutMs, err := protocol.ByteToBase10(params[2]) if err != nil { return nil, protocol.NewFatalClientErr(err, "E_INVALID", fmt.Sprintf("DPUB could not parse defer timeout %s", params[2])) } timeoutDuration := time.Duration(timeoutMs) * time.Millisecond if timeoutDuration < 0 || timeoutDuration > p.nsqd.getOpts().MaxDeferTimeout { return nil, protocol.NewFatalClientErr(nil, "E_INVALID", fmt.Sprintf("DPUB defer timeout %d out of range 0-%d", timeoutMs, p.nsqd.getOpts().MaxDeferTimeout/time.Millisecond)) } bodyLen, err := readLen(client.Reader, client.lenSlice) if err != nil { return nil, protocol.NewFatalClientErr(err, "E_BAD_MESSAGE", "DPUB failed to read message body size") } if bodyLen <= 0 { return nil, protocol.NewFatalClientErr(nil, "E_BAD_MESSAGE", fmt.Sprintf("DPUB invalid message body size %d", bodyLen)) } if int64(bodyLen) > p.nsqd.getOpts().MaxMsgSize { return nil, protocol.NewFatalClientErr(nil, "E_BAD_MESSAGE", fmt.Sprintf("DPUB message too big %d > %d", bodyLen, p.nsqd.getOpts().MaxMsgSize)) } messageBody := make([]byte, bodyLen) _, err = io.ReadFull(client.Reader, messageBody) if err != nil { return nil, protocol.NewFatalClientErr(err, "E_BAD_MESSAGE", "DPUB failed to read message body") } if err := p.CheckAuth(client, "DPUB", topicName, ""); err != nil { return nil, err } topic := p.nsqd.GetTopic(topicName) msg := NewMessage(topic.GenerateID(), messageBody) msg.deferred = timeoutDuration err = topic.PutMessage(msg) if err != nil { return nil, protocol.NewFatalClientErr(err, "E_DPUB_FAILED", "DPUB failed "+err.Error()) } client.PublishedMessage(topicName, 1) return okBytes, nil } func (p *protocolV2) TOUCH(client *clientV2, params [][]byte) ([]byte, error) { state := atomic.LoadInt32(&client.State) if state != stateSubscribed && state != stateClosing { return nil, protocol.NewFatalClientErr(nil, "E_INVALID", "cannot TOUCH in current state") } if len(params) < 2 { return nil, protocol.NewFatalClientErr(nil, "E_INVALID", "TOUCH insufficient number of params") } id, err := getMessageID(params[1]) if err != nil { return nil, protocol.NewFatalClientErr(nil, "E_INVALID", err.Error()) } client.writeLock.RLock() msgTimeout := client.MsgTimeout client.writeLock.RUnlock() err = client.Channel.TouchMessage(client.ID, *id, msgTimeout) if err != nil { return nil, protocol.NewClientErr(err, "E_TOUCH_FAILED", fmt.Sprintf("TOUCH %s failed %s", *id, err.Error())) } return nil, nil } func readMPUB(r io.Reader, tmp []byte, topic *Topic, maxMessageSize int64, maxBodySize int64) ([]*Message, error) { numMessages, err := readLen(r, tmp) if err != nil { return nil, protocol.NewFatalClientErr(err, "E_BAD_BODY", "MPUB failed to read message count") } // 4 == total num, 5 == length + min 1 maxMessages := (maxBodySize - 4) / 5 if numMessages <= 0 || int64(numMessages) > maxMessages { return nil, protocol.NewFatalClientErr(err, "E_BAD_BODY", fmt.Sprintf("MPUB invalid message count %d", numMessages)) } messages := make([]*Message, 0, numMessages) for i := int32(0); i < numMessages; i++ { messageSize, err := readLen(r, tmp) if err != nil { return nil, protocol.NewFatalClientErr(err, "E_BAD_MESSAGE", fmt.Sprintf("MPUB failed to read message(%d) body size", i)) } if messageSize <= 0 { return nil, protocol.NewFatalClientErr(nil, "E_BAD_MESSAGE", fmt.Sprintf("MPUB invalid message(%d) body size %d", i, messageSize)) } if int64(messageSize) > maxMessageSize { return nil, protocol.NewFatalClientErr(nil, "E_BAD_MESSAGE", fmt.Sprintf("MPUB message too big %d > %d", messageSize, maxMessageSize)) } msgBody := make([]byte, messageSize) _, err = io.ReadFull(r, msgBody) if err != nil { return nil, protocol.NewFatalClientErr(err, "E_BAD_MESSAGE", "MPUB failed to read message body") } messages = append(messages, NewMessage(topic.GenerateID(), msgBody)) } return messages, nil } // validate and cast the bytes on the wire to a message ID func getMessageID(p []byte) (*MessageID, error) { if len(p) != MsgIDLength { return nil, errors.New("invalid message ID") } return (*MessageID)(unsafe.Pointer(&p[0])), nil } func readLen(r io.Reader, tmp []byte) (int32, error) { _, err := io.ReadFull(r, tmp) if err != nil { return 0, err } return int32(binary.BigEndian.Uint32(tmp)), nil } func enforceTLSPolicy(client *clientV2, p *protocolV2, command []byte) error { if p.nsqd.getOpts().TLSRequired != TLSNotRequired && atomic.LoadInt32(&client.TLS) != 1 { return protocol.NewFatalClientErr(nil, "E_INVALID", fmt.Sprintf("cannot %s in current state (TLS required)", command)) } return nil } ================================================ FILE: nsqd/protocol_v2_test.go ================================================ package nsqd import ( "bufio" "bytes" "compress/flate" "crypto/tls" "encoding/json" "errors" "fmt" "io" "math" "math/rand" "net" "net/http" "net/http/httptest" "net/url" "os" "runtime" "strconv" "strings" "sync" "sync/atomic" "testing" "time" "github.com/golang/snappy" "github.com/nsqio/go-nsq" "github.com/nsqio/nsq/internal/protocol" "github.com/nsqio/nsq/internal/test" ) func mustStartNSQD(opts *Options) (net.Addr, net.Addr, *NSQD) { opts.TCPAddress = "127.0.0.1:0" opts.HTTPAddress = "127.0.0.1:0" opts.HTTPSAddress = "127.0.0.1:0" if opts.DataPath == "" { tmpDir, err := os.MkdirTemp("", "nsq-test-") if err != nil { panic(err) } opts.DataPath = tmpDir } nsqd, err := New(opts) if err != nil { panic(err) } go func() { err := nsqd.Main() if err != nil { panic(err) } }() return nsqd.RealTCPAddr(), nsqd.RealHTTPAddr(), nsqd } func mustConnectNSQD(tcpAddr net.Addr) (net.Conn, error) { conn, err := net.DialTimeout("tcp", tcpAddr.String(), time.Second) if err != nil { return nil, err } conn.Write(nsq.MagicV2) return conn, nil } func identify(t *testing.T, conn io.ReadWriter, extra map[string]interface{}, f int32) []byte { ci := make(map[string]interface{}) ci["client_id"] = "test" ci["feature_negotiation"] = true for k, v := range extra { ci[k] = v } cmd, _ := nsq.Identify(ci) _, err := cmd.WriteTo(conn) test.Nil(t, err) resp, err := nsq.ReadResponse(conn) test.Nil(t, err) frameType, data, err := nsq.UnpackResponse(resp) test.Nil(t, err) test.Equal(t, frameType, f) return data } func sub(t *testing.T, conn io.ReadWriter, topicName string, channelName string) { _, err := nsq.Subscribe(topicName, channelName).WriteTo(conn) test.Nil(t, err) readValidate(t, conn, frameTypeResponse, "OK") } func authCmd(t *testing.T, conn io.ReadWriter, authSecret string, expectSuccess string) { auth, _ := nsq.Auth(authSecret) _, err := auth.WriteTo(conn) test.Nil(t, err) if expectSuccess != "" { readValidate(t, conn, nsq.FrameTypeResponse, expectSuccess) } } func subFail(t *testing.T, conn io.ReadWriter, topicName string, channelName string) { _, err := nsq.Subscribe(topicName, channelName).WriteTo(conn) test.Nil(t, err) resp, _ := nsq.ReadResponse(conn) frameType, _, _ := nsq.UnpackResponse(resp) test.Equal(t, frameTypeError, frameType) } func readValidate(t *testing.T, conn io.Reader, f int32, d string) []byte { resp, err := nsq.ReadResponse(conn) test.Nil(t, err) frameType, data, err := nsq.UnpackResponse(resp) test.Nil(t, err) test.Equal(t, f, frameType) test.Equal(t, d, string(data)) return data } // test channel/topic names func TestChannelTopicNames(t *testing.T) { test.Equal(t, true, protocol.IsValidChannelName("test")) test.Equal(t, true, protocol.IsValidChannelName("test-with_period.")) test.Equal(t, true, protocol.IsValidChannelName("test#ephemeral")) test.Equal(t, true, protocol.IsValidTopicName("test")) test.Equal(t, true, protocol.IsValidTopicName("test-with_period.")) test.Equal(t, true, protocol.IsValidTopicName("test#ephemeral")) test.Equal(t, false, protocol.IsValidTopicName("test:ephemeral")) } // exercise the basic operations of the V2 protocol func TestBasicV2(t *testing.T) { opts := NewOptions() opts.Logger = test.NewTestLogger(t) opts.ClientTimeout = 60 * time.Second tcpAddr, _, nsqd := mustStartNSQD(opts) defer os.RemoveAll(opts.DataPath) defer nsqd.Exit() topicName := "test_v2" + strconv.Itoa(int(time.Now().Unix())) topic := nsqd.GetTopic(topicName) msg := NewMessage(topic.GenerateID(), []byte("test body")) topic.PutMessage(msg) conn, err := mustConnectNSQD(tcpAddr) test.Nil(t, err) defer conn.Close() identify(t, conn, nil, frameTypeResponse) sub(t, conn, topicName, "ch") _, err = nsq.Ready(1).WriteTo(conn) test.Nil(t, err) resp, err := nsq.ReadResponse(conn) test.Nil(t, err) frameType, data, _ := nsq.UnpackResponse(resp) msgOut, _ := decodeMessage(data) test.Equal(t, frameTypeMessage, frameType) test.Equal(t, msg.ID, msgOut.ID) test.Equal(t, msg.Body, msgOut.Body) test.Equal(t, uint16(1), msgOut.Attempts) } func TestMultipleConsumerV2(t *testing.T) { msgChan := make(chan *Message) opts := NewOptions() opts.Logger = test.NewTestLogger(t) opts.ClientTimeout = 60 * time.Second tcpAddr, _, nsqd := mustStartNSQD(opts) defer os.RemoveAll(opts.DataPath) defer nsqd.Exit() topicName := "test_multiple_v2" + strconv.Itoa(int(time.Now().Unix())) topic := nsqd.GetTopic(topicName) msg := NewMessage(topic.GenerateID(), []byte("test body")) topic.GetChannel("ch1") topic.GetChannel("ch2") topic.PutMessage(msg) for _, i := range []string{"1", "2"} { conn, err := mustConnectNSQD(tcpAddr) test.Nil(t, err) defer conn.Close() identify(t, conn, nil, frameTypeResponse) sub(t, conn, topicName, "ch"+i) _, err = nsq.Ready(1).WriteTo(conn) test.Nil(t, err) go func(c net.Conn) { resp, err := nsq.ReadResponse(c) test.Nil(t, err) _, data, err := nsq.UnpackResponse(resp) test.Nil(t, err) msg, err := decodeMessage(data) test.Nil(t, err) msgChan <- msg }(conn) } msgOut := <-msgChan test.Equal(t, msg.ID, msgOut.ID) test.Equal(t, msg.Body, msgOut.Body) test.Equal(t, uint16(1), msgOut.Attempts) msgOut = <-msgChan test.Equal(t, msg.ID, msgOut.ID) test.Equal(t, msg.Body, msgOut.Body) test.Equal(t, uint16(1), msgOut.Attempts) } // TestSameZoneConsumerV2 tests that a published message goes to same-zone consumer first // if it's message pump is waiting func TestSameZoneConsumerV2(t *testing.T) { opts := NewOptions() opts.Experiments = []string{string(TopologyAwareConsumption)} opts.Logger = test.NewTestLogger(t) opts.ClientTimeout = 60 * time.Second opts.TopologyRegion = "region" opts.TopologyZone = "zone" tcpAddr, _, nsqd := mustStartNSQD(opts) defer os.RemoveAll(opts.DataPath) defer nsqd.Exit() topicName := "test_zone_v2" + strconv.Itoa(int(time.Now().Unix())) topic := nsqd.GetTopic(topicName) topic.GetChannel("ch") var sameZone, diffZone int64 var exiting int32 done := make(chan bool, 21) for _, zone := range []string{"zone", "zone", "zone2", "zone2"} { zone := zone conn, err := mustConnectNSQD(tcpAddr) test.Nil(t, err) defer conn.Close() identify(t, conn, map[string]interface{}{"topology_zone": zone}, frameTypeResponse) sub(t, conn, topicName, "ch") _, err = nsq.Ready(10).WriteTo(conn) test.Nil(t, err) go func(c net.Conn, zone string) { for { resp, err := nsq.ReadResponse(c) if atomic.LoadInt32(&exiting) == 1 { return } test.Nil(t, err) _, data, err := nsq.UnpackResponse(resp) test.Nil(t, err) _, err = decodeMessage(data) test.Nil(t, err) if zone == "zone" { atomic.AddInt64(&sameZone, 1) } else { atomic.AddInt64(&diffZone, 1) } done <- true } }(conn, zone) } // first 20 messages go to same zone (each has RDY 10) // next message goes to global memoryChan (All consumers) for i := 0; i < 21; i++ { topic.PutMessage(NewMessage(topic.GenerateID(), make([]byte, 100))) if i%2 == 0 { // sleep long enough for messagePump to wait again time.Sleep(time.Millisecond) } } var doneCount int64 for range done { doneCount += 1 if doneCount == 21 { break } } t.Logf("got same zone %d diffZone %d", sameZone, diffZone) atomic.StoreInt32(&exiting, 1) test.Equal(t, int64(20), sameZone) test.Equal(t, int64(1), diffZone) } func TestClientTimeout(t *testing.T) { topicName := "test_client_timeout_v2" + strconv.Itoa(int(time.Now().Unix())) opts := NewOptions() opts.Logger = test.NewTestLogger(t) opts.ClientTimeout = 150 * time.Millisecond opts.LogLevel = LOG_DEBUG tcpAddr, _, nsqd := mustStartNSQD(opts) defer os.RemoveAll(opts.DataPath) defer nsqd.Exit() conn, err := mustConnectNSQD(tcpAddr) test.Nil(t, err) defer conn.Close() identify(t, conn, nil, frameTypeResponse) sub(t, conn, topicName, "ch") time.Sleep(150 * time.Millisecond) // depending on timing there may be 1 or 2 hearbeats sent // just read until we get an error timer := time.After(100 * time.Millisecond) for { select { case <-timer: t.Fatalf("test timed out") default: _, err := nsq.ReadResponse(conn) if err != nil { goto done } } } done: } func TestClientHeartbeat(t *testing.T) { topicName := "test_hb_v2" + strconv.Itoa(int(time.Now().Unix())) opts := NewOptions() opts.Logger = test.NewTestLogger(t) opts.ClientTimeout = 200 * time.Millisecond tcpAddr, _, nsqd := mustStartNSQD(opts) defer os.RemoveAll(opts.DataPath) defer nsqd.Exit() conn, err := mustConnectNSQD(tcpAddr) test.Nil(t, err) defer conn.Close() identify(t, conn, nil, frameTypeResponse) sub(t, conn, topicName, "ch") _, err = nsq.Ready(1).WriteTo(conn) test.Nil(t, err) resp, _ := nsq.ReadResponse(conn) _, data, _ := nsq.UnpackResponse(resp) test.Equal(t, []byte("_heartbeat_"), data) time.Sleep(20 * time.Millisecond) _, err = nsq.Nop().WriteTo(conn) test.Nil(t, err) // wait long enough that would have timed out (had we not sent the above cmd) time.Sleep(100 * time.Millisecond) _, err = nsq.Nop().WriteTo(conn) test.Nil(t, err) } func TestClientHeartbeatDisableSUB(t *testing.T) { topicName := "test_hb_v2" + strconv.Itoa(int(time.Now().Unix())) opts := NewOptions() opts.Logger = test.NewTestLogger(t) opts.ClientTimeout = 200 * time.Millisecond opts.LogLevel = LOG_DEBUG tcpAddr, _, nsqd := mustStartNSQD(opts) defer os.RemoveAll(opts.DataPath) defer nsqd.Exit() conn, err := mustConnectNSQD(tcpAddr) test.Nil(t, err) defer conn.Close() identify(t, conn, map[string]interface{}{ "heartbeat_interval": -1, }, frameTypeResponse) subFail(t, conn, topicName, "ch") } func TestClientHeartbeatDisable(t *testing.T) { opts := NewOptions() opts.Logger = test.NewTestLogger(t) opts.ClientTimeout = 100 * time.Millisecond tcpAddr, _, nsqd := mustStartNSQD(opts) defer os.RemoveAll(opts.DataPath) defer nsqd.Exit() conn, err := mustConnectNSQD(tcpAddr) test.Nil(t, err) defer conn.Close() identify(t, conn, map[string]interface{}{ "heartbeat_interval": -1, }, frameTypeResponse) time.Sleep(150 * time.Millisecond) _, err = nsq.Nop().WriteTo(conn) test.Nil(t, err) } func TestMaxHeartbeatIntervalValid(t *testing.T) { opts := NewOptions() opts.Logger = test.NewTestLogger(t) opts.MaxHeartbeatInterval = 300 * time.Second tcpAddr, _, nsqd := mustStartNSQD(opts) defer os.RemoveAll(opts.DataPath) defer nsqd.Exit() conn, err := mustConnectNSQD(tcpAddr) test.Nil(t, err) defer conn.Close() hbi := int(opts.MaxHeartbeatInterval / time.Millisecond) identify(t, conn, map[string]interface{}{ "heartbeat_interval": hbi, }, frameTypeResponse) } func TestMaxHeartbeatIntervalInvalid(t *testing.T) { opts := NewOptions() opts.Logger = test.NewTestLogger(t) opts.MaxHeartbeatInterval = 300 * time.Second tcpAddr, _, nsqd := mustStartNSQD(opts) defer os.RemoveAll(opts.DataPath) defer nsqd.Exit() conn, err := mustConnectNSQD(tcpAddr) test.Nil(t, err) defer conn.Close() hbi := int(opts.MaxHeartbeatInterval/time.Millisecond + 1) data := identify(t, conn, map[string]interface{}{ "heartbeat_interval": hbi, }, frameTypeError) test.Equal(t, "E_BAD_BODY IDENTIFY heartbeat interval (300001) is invalid", string(data)) } func TestPausing(t *testing.T) { topicName := "test_pause_v2" + strconv.Itoa(int(time.Now().Unix())) opts := NewOptions() opts.Logger = test.NewTestLogger(t) tcpAddr, _, nsqd := mustStartNSQD(opts) defer os.RemoveAll(opts.DataPath) defer nsqd.Exit() conn, err := mustConnectNSQD(tcpAddr) test.Nil(t, err) defer conn.Close() identify(t, conn, nil, frameTypeResponse) sub(t, conn, topicName, "ch") _, err = nsq.Ready(1).WriteTo(conn) test.Nil(t, err) topic := nsqd.GetTopic(topicName) msg := NewMessage(topic.GenerateID(), []byte("test body")) channel := topic.GetChannel("ch") topic.PutMessage(msg) // receive the first message via the client, finish it, and send new RDY resp, _ := nsq.ReadResponse(conn) _, data, _ := nsq.UnpackResponse(resp) msg, _ = decodeMessage(data) test.Equal(t, []byte("test body"), msg.Body) _, err = nsq.Finish(nsq.MessageID(msg.ID)).WriteTo(conn) test.Nil(t, err) _, err = nsq.Ready(1).WriteTo(conn) test.Nil(t, err) // sleep to allow the RDY state to take effect time.Sleep(50 * time.Millisecond) // pause the channel... the client shouldn't receive any more messages channel.Pause() // sleep to allow the paused state to take effect time.Sleep(50 * time.Millisecond) msg = NewMessage(topic.GenerateID(), []byte("test body2")) topic.PutMessage(msg) // allow the client to possibly get a message, the test would hang indefinitely // if pausing was not working time.Sleep(50 * time.Millisecond) msg = <-channel.memoryMsgChan test.Equal(t, []byte("test body2"), msg.Body) // unpause the channel... the client should now be pushed a message channel.UnPause() msg = NewMessage(topic.GenerateID(), []byte("test body3")) topic.PutMessage(msg) resp, _ = nsq.ReadResponse(conn) _, data, _ = nsq.UnpackResponse(resp) msg, _ = decodeMessage(data) test.Equal(t, []byte("test body3"), msg.Body) } func TestEmptyCommand(t *testing.T) { opts := NewOptions() opts.Logger = test.NewTestLogger(t) tcpAddr, _, nsqd := mustStartNSQD(opts) defer os.RemoveAll(opts.DataPath) defer nsqd.Exit() conn, err := mustConnectNSQD(tcpAddr) test.Nil(t, err) defer conn.Close() _, err = conn.Write([]byte("\n\n")) test.Nil(t, err) // if we didn't panic here we're good, see issue #120 } func TestSizeLimits(t *testing.T) { opts := NewOptions() opts.Logger = test.NewTestLogger(t) opts.LogLevel = LOG_DEBUG opts.MaxMsgSize = 100 opts.MaxBodySize = 1000 tcpAddr, _, nsqd := mustStartNSQD(opts) defer os.RemoveAll(opts.DataPath) defer nsqd.Exit() conn, err := mustConnectNSQD(tcpAddr) test.Nil(t, err) defer conn.Close() topicName := "test_limits_v2" + strconv.Itoa(int(time.Now().Unix())) identify(t, conn, nil, frameTypeResponse) sub(t, conn, topicName, "ch") // PUB that's valid nsq.Publish(topicName, make([]byte, 95)).WriteTo(conn) resp, _ := nsq.ReadResponse(conn) frameType, data, _ := nsq.UnpackResponse(resp) t.Logf("frameType: %d, data: %s", frameType, data) test.Equal(t, frameTypeResponse, frameType) test.Equal(t, []byte("OK"), data) // PUB that's invalid (too big) nsq.Publish(topicName, make([]byte, 105)).WriteTo(conn) resp, _ = nsq.ReadResponse(conn) frameType, data, _ = nsq.UnpackResponse(resp) t.Logf("frameType: %d, data: %s", frameType, data) test.Equal(t, frameTypeError, frameType) test.Equal(t, "E_BAD_MESSAGE PUB message too big 105 > 100", string(data)) // need to reconnect conn, err = mustConnectNSQD(tcpAddr) test.Nil(t, err) defer conn.Close() // PUB thats empty nsq.Publish(topicName, []byte{}).WriteTo(conn) resp, _ = nsq.ReadResponse(conn) frameType, data, _ = nsq.UnpackResponse(resp) t.Logf("frameType: %d, data: %s", frameType, data) test.Equal(t, frameTypeError, frameType) test.Equal(t, "E_BAD_MESSAGE PUB invalid message body size 0", string(data)) // need to reconnect conn, err = mustConnectNSQD(tcpAddr) test.Nil(t, err) defer conn.Close() // MPUB body that's valid mpub := make([][]byte, 5) for i := range mpub { mpub[i] = make([]byte, 100) } cmd, _ := nsq.MultiPublish(topicName, mpub) cmd.WriteTo(conn) resp, _ = nsq.ReadResponse(conn) frameType, data, _ = nsq.UnpackResponse(resp) t.Logf("frameType: %d, data: %s", frameType, data) test.Equal(t, frameTypeResponse, frameType) test.Equal(t, []byte("OK"), data) // MPUB body that's invalid (body too big) mpub = make([][]byte, 11) for i := range mpub { mpub[i] = make([]byte, 100) } cmd, _ = nsq.MultiPublish(topicName, mpub) cmd.WriteTo(conn) resp, _ = nsq.ReadResponse(conn) frameType, data, _ = nsq.UnpackResponse(resp) t.Logf("frameType: %d, data: %s", frameType, data) test.Equal(t, frameTypeError, frameType) test.Equal(t, "E_BAD_BODY MPUB body too big 1148 > 1000", string(data)) // need to reconnect conn, err = mustConnectNSQD(tcpAddr) test.Nil(t, err) defer conn.Close() // MPUB that's invalid (one message empty) mpub = make([][]byte, 5) for i := range mpub { mpub[i] = make([]byte, 100) } mpub = append(mpub, []byte{}) cmd, _ = nsq.MultiPublish(topicName, mpub) cmd.WriteTo(conn) resp, _ = nsq.ReadResponse(conn) frameType, data, _ = nsq.UnpackResponse(resp) t.Logf("frameType: %d, data: %s", frameType, data) test.Equal(t, frameTypeError, frameType) test.Equal(t, "E_BAD_MESSAGE MPUB invalid message(5) body size 0", string(data)) // need to reconnect conn, err = mustConnectNSQD(tcpAddr) test.Nil(t, err) defer conn.Close() // MPUB body that's invalid (one of the messages is too big) mpub = make([][]byte, 5) for i := range mpub { mpub[i] = make([]byte, 101) } cmd, _ = nsq.MultiPublish(topicName, mpub) cmd.WriteTo(conn) resp, _ = nsq.ReadResponse(conn) frameType, data, _ = nsq.UnpackResponse(resp) t.Logf("frameType: %d, data: %s", frameType, data) test.Equal(t, frameTypeError, frameType) test.Equal(t, "E_BAD_MESSAGE MPUB message too big 101 > 100", string(data)) } func TestDPUB(t *testing.T) { opts := NewOptions() opts.Logger = test.NewTestLogger(t) opts.LogLevel = LOG_DEBUG tcpAddr, _, nsqd := mustStartNSQD(opts) defer os.RemoveAll(opts.DataPath) defer nsqd.Exit() conn, err := mustConnectNSQD(tcpAddr) test.Nil(t, err) defer conn.Close() topicName := "test_dpub_v2" + strconv.Itoa(int(time.Now().Unix())) identify(t, conn, nil, frameTypeResponse) sub(t, conn, topicName, "ch") // valid nsq.DeferredPublish(topicName, time.Second, make([]byte, 100)).WriteTo(conn) resp, _ := nsq.ReadResponse(conn) frameType, data, _ := nsq.UnpackResponse(resp) t.Logf("frameType: %d, data: %s", frameType, data) test.Equal(t, frameTypeResponse, frameType) test.Equal(t, []byte("OK"), data) time.Sleep(25 * time.Millisecond) ch := nsqd.GetTopic(topicName).GetChannel("ch") ch.deferredMutex.Lock() numDef := len(ch.deferredMessages) ch.deferredMutex.Unlock() test.Equal(t, 1, numDef) test.Equal(t, 1, int(atomic.LoadUint64(&ch.messageCount))) // duration out of range nsq.DeferredPublish(topicName, opts.MaxDeferTimeout+100*time.Millisecond, make([]byte, 100)).WriteTo(conn) resp, _ = nsq.ReadResponse(conn) frameType, data, _ = nsq.UnpackResponse(resp) t.Logf("frameType: %d, data: %s", frameType, data) test.Equal(t, frameTypeError, frameType) test.Equal(t, "E_INVALID DPUB defer timeout 3600100 out of range 0-3600000", string(data)) } func TestTouch(t *testing.T) { opts := NewOptions() opts.Logger = test.NewTestLogger(t) opts.LogLevel = LOG_DEBUG opts.MsgTimeout = 150 * time.Millisecond tcpAddr, _, nsqd := mustStartNSQD(opts) defer os.RemoveAll(opts.DataPath) defer nsqd.Exit() topicName := "test_touch" + strconv.Itoa(int(time.Now().Unix())) conn, err := mustConnectNSQD(tcpAddr) test.Nil(t, err) defer conn.Close() identify(t, conn, nil, frameTypeResponse) sub(t, conn, topicName, "ch") topic := nsqd.GetTopic(topicName) channel := topic.GetChannel("ch") msg := NewMessage(topic.GenerateID(), []byte("test body")) topic.PutMessage(msg) _, err = nsq.Ready(1).WriteTo(conn) test.Nil(t, err) resp, err := nsq.ReadResponse(conn) test.Nil(t, err) frameType, data, _ := nsq.UnpackResponse(resp) msgOut, _ := decodeMessage(data) test.Equal(t, frameTypeMessage, frameType) test.Equal(t, msg.ID, msgOut.ID) time.Sleep(75 * time.Millisecond) _, err = nsq.Touch(nsq.MessageID(msg.ID)).WriteTo(conn) test.Nil(t, err) time.Sleep(75 * time.Millisecond) _, err = nsq.Finish(nsq.MessageID(msg.ID)).WriteTo(conn) test.Nil(t, err) test.Equal(t, uint64(0), channel.timeoutCount) } func TestMaxRdyCount(t *testing.T) { opts := NewOptions() opts.Logger = test.NewTestLogger(t) opts.LogLevel = LOG_DEBUG opts.MaxRdyCount = 50 tcpAddr, _, nsqd := mustStartNSQD(opts) defer os.RemoveAll(opts.DataPath) defer nsqd.Exit() topicName := "test_max_rdy_count" + strconv.Itoa(int(time.Now().Unix())) conn, err := mustConnectNSQD(tcpAddr) test.Nil(t, err) defer conn.Close() topic := nsqd.GetTopic(topicName) msg := NewMessage(topic.GenerateID(), []byte("test body")) topic.PutMessage(msg) data := identify(t, conn, nil, frameTypeResponse) r := struct { MaxRdyCount int64 `json:"max_rdy_count"` }{} err = json.Unmarshal(data, &r) test.Nil(t, err) test.Equal(t, int64(50), r.MaxRdyCount) sub(t, conn, topicName, "ch") _, err = nsq.Ready(int(opts.MaxRdyCount)).WriteTo(conn) test.Nil(t, err) resp, err := nsq.ReadResponse(conn) test.Nil(t, err) frameType, data, _ := nsq.UnpackResponse(resp) msgOut, _ := decodeMessage(data) test.Equal(t, frameTypeMessage, frameType) test.Equal(t, msg.ID, msgOut.ID) _, err = nsq.Ready(int(opts.MaxRdyCount) + 1).WriteTo(conn) test.Nil(t, err) resp, err = nsq.ReadResponse(conn) test.Nil(t, err) frameType, data, _ = nsq.UnpackResponse(resp) test.Equal(t, int32(1), frameType) test.Equal(t, "E_INVALID RDY count 51 out of range 0-50", string(data)) } func TestFatalError(t *testing.T) { opts := NewOptions() opts.Logger = test.NewTestLogger(t) tcpAddr, _, nsqd := mustStartNSQD(opts) defer os.RemoveAll(opts.DataPath) defer nsqd.Exit() conn, err := mustConnectNSQD(tcpAddr) test.Nil(t, err) defer conn.Close() _, err = conn.Write([]byte("ASDF\n")) test.Nil(t, err) resp, err := nsq.ReadResponse(conn) test.Nil(t, err) frameType, data, _ := nsq.UnpackResponse(resp) test.Equal(t, int32(1), frameType) test.Equal(t, "E_INVALID invalid command ASDF", string(data)) _, err = nsq.ReadResponse(conn) test.NotNil(t, err) } func TestOutputBuffering(t *testing.T) { opts := NewOptions() opts.Logger = test.NewTestLogger(t) opts.LogLevel = LOG_DEBUG opts.MaxOutputBufferSize = 512 * 1024 opts.MaxOutputBufferTimeout = time.Second tcpAddr, _, nsqd := mustStartNSQD(opts) defer os.RemoveAll(opts.DataPath) defer nsqd.Exit() topicName := "test_output_buffering" + strconv.Itoa(int(time.Now().Unix())) conn, err := mustConnectNSQD(tcpAddr) test.Nil(t, err) defer conn.Close() outputBufferSize := 256 * 1024 outputBufferTimeout := 500 topic := nsqd.GetTopic(topicName) msg := NewMessage(topic.GenerateID(), make([]byte, outputBufferSize-1024)) topic.PutMessage(msg) start := time.Now() data := identify(t, conn, map[string]interface{}{ "output_buffer_size": outputBufferSize, "output_buffer_timeout": outputBufferTimeout, }, frameTypeResponse) var decoded map[string]interface{} json.Unmarshal(data, &decoded) v, ok := decoded["output_buffer_size"] test.Equal(t, true, ok) test.Equal(t, outputBufferSize, int(v.(float64))) v = decoded["output_buffer_timeout"] test.Equal(t, outputBufferTimeout, int(v.(float64))) sub(t, conn, topicName, "ch") _, err = nsq.Ready(10).WriteTo(conn) test.Nil(t, err) resp, err := nsq.ReadResponse(conn) test.Nil(t, err) end := time.Now() test.Equal(t, true, int(end.Sub(start)/time.Millisecond) >= outputBufferTimeout) frameType, data, _ := nsq.UnpackResponse(resp) msgOut, _ := decodeMessage(data) test.Equal(t, frameTypeMessage, frameType) test.Equal(t, msg.ID, msgOut.ID) } func TestOutputBufferingValidity(t *testing.T) { opts := NewOptions() opts.Logger = test.NewTestLogger(t) opts.LogLevel = LOG_DEBUG opts.MaxOutputBufferSize = 512 * 1024 opts.MaxOutputBufferTimeout = time.Second tcpAddr, _, nsqd := mustStartNSQD(opts) defer os.RemoveAll(opts.DataPath) defer nsqd.Exit() conn, err := mustConnectNSQD(tcpAddr) test.Nil(t, err) defer conn.Close() identify(t, conn, map[string]interface{}{ "output_buffer_size": 512 * 1024, "output_buffer_timeout": 1000, }, frameTypeResponse) identify(t, conn, map[string]interface{}{ "output_buffer_size": -1, "output_buffer_timeout": -1, }, frameTypeResponse) identify(t, conn, map[string]interface{}{ "output_buffer_size": 0, "output_buffer_timeout": 0, }, frameTypeResponse) data := identify(t, conn, map[string]interface{}{ "output_buffer_size": 512*1024 + 1, "output_buffer_timeout": 0, }, frameTypeError) test.Equal(t, fmt.Sprintf("E_BAD_BODY IDENTIFY output buffer size (%d) is invalid", 512*1024+1), string(data)) conn, err = mustConnectNSQD(tcpAddr) test.Nil(t, err) defer conn.Close() data = identify(t, conn, map[string]interface{}{ "output_buffer_size": 0, "output_buffer_timeout": 1001, }, frameTypeError) test.Equal(t, "E_BAD_BODY IDENTIFY output buffer timeout (1001) is invalid", string(data)) } func TestTLS(t *testing.T) { opts := NewOptions() opts.Logger = test.NewTestLogger(t) opts.LogLevel = LOG_DEBUG opts.TLSCert = "./test/certs/server.pem" opts.TLSKey = "./test/certs/server.key" tcpAddr, _, nsqd := mustStartNSQD(opts) defer os.RemoveAll(opts.DataPath) defer nsqd.Exit() conn, err := mustConnectNSQD(tcpAddr) test.Nil(t, err) defer conn.Close() data := identify(t, conn, map[string]interface{}{ "tls_v1": true, }, frameTypeResponse) r := struct { TLSv1 bool `json:"tls_v1"` }{} err = json.Unmarshal(data, &r) test.Nil(t, err) test.Equal(t, true, r.TLSv1) tlsConfig := &tls.Config{ InsecureSkipVerify: true, } tlsConn := tls.Client(conn, tlsConfig) err = tlsConn.Handshake() test.Nil(t, err) resp, _ := nsq.ReadResponse(tlsConn) frameType, data, _ := nsq.UnpackResponse(resp) t.Logf("frameType: %d, data: %s", frameType, data) test.Equal(t, frameTypeResponse, frameType) test.Equal(t, []byte("OK"), data) } func TestTLSRequired(t *testing.T) { opts := NewOptions() opts.Logger = test.NewTestLogger(t) opts.LogLevel = LOG_DEBUG opts.TLSCert = "./test/certs/server.pem" opts.TLSKey = "./test/certs/server.key" opts.TLSRequired = TLSRequiredExceptHTTP tcpAddr, _, nsqd := mustStartNSQD(opts) defer os.RemoveAll(opts.DataPath) defer nsqd.Exit() topicName := "test_tls_required" + strconv.Itoa(int(time.Now().Unix())) conn, err := mustConnectNSQD(tcpAddr) test.Nil(t, err) defer conn.Close() subFail(t, conn, topicName, "ch") conn, err = mustConnectNSQD(tcpAddr) test.Nil(t, err) defer conn.Close() data := identify(t, conn, map[string]interface{}{ "tls_v1": true, }, frameTypeResponse) r := struct { TLSv1 bool `json:"tls_v1"` }{} err = json.Unmarshal(data, &r) test.Nil(t, err) test.Equal(t, true, r.TLSv1) tlsConfig := &tls.Config{ InsecureSkipVerify: true, } tlsConn := tls.Client(conn, tlsConfig) err = tlsConn.Handshake() test.Nil(t, err) resp, _ := nsq.ReadResponse(tlsConn) frameType, data, _ := nsq.UnpackResponse(resp) t.Logf("frameType: %d, data: %s", frameType, data) test.Equal(t, frameTypeResponse, frameType) test.Equal(t, []byte("OK"), data) } func TestTLSAuthRequire(t *testing.T) { opts := NewOptions() opts.Logger = test.NewTestLogger(t) opts.LogLevel = LOG_DEBUG opts.TLSCert = "./test/certs/server.pem" opts.TLSKey = "./test/certs/server.key" opts.TLSClientAuthPolicy = "require" tcpAddr, _, nsqd := mustStartNSQD(opts) defer os.RemoveAll(opts.DataPath) defer nsqd.Exit() // No Certs conn, err := mustConnectNSQD(tcpAddr) test.Nil(t, err) defer conn.Close() data := identify(t, conn, map[string]interface{}{ "tls_v1": true, }, frameTypeResponse) r := struct { TLSv1 bool `json:"tls_v1"` }{} err = json.Unmarshal(data, &r) test.Nil(t, err) test.Equal(t, true, r.TLSv1) tlsConfig := &tls.Config{ InsecureSkipVerify: true, } tlsConn := tls.Client(conn, tlsConfig) _, err = nsq.ReadResponse(tlsConn) test.NotNil(t, err) // With Unsigned Cert conn, err = mustConnectNSQD(tcpAddr) test.Nil(t, err) defer conn.Close() data = identify(t, conn, map[string]interface{}{ "tls_v1": true, }, frameTypeResponse) r = struct { TLSv1 bool `json:"tls_v1"` }{} err = json.Unmarshal(data, &r) test.Nil(t, err) test.Equal(t, true, r.TLSv1) cert, err := tls.LoadX509KeyPair("./test/certs/cert.pem", "./test/certs/key.pem") test.Nil(t, err) tlsConfig = &tls.Config{ Certificates: []tls.Certificate{cert}, InsecureSkipVerify: true, } tlsConn = tls.Client(conn, tlsConfig) err = tlsConn.Handshake() test.Nil(t, err) resp, _ := nsq.ReadResponse(tlsConn) frameType, data, _ := nsq.UnpackResponse(resp) t.Logf("frameType: %d, data: %s", frameType, data) test.Equal(t, frameTypeResponse, frameType) test.Equal(t, []byte("OK"), data) } func TestTLSAuthRequireVerify(t *testing.T) { opts := NewOptions() opts.Logger = test.NewTestLogger(t) opts.LogLevel = LOG_DEBUG opts.TLSCert = "./test/certs/server.pem" opts.TLSKey = "./test/certs/server.key" opts.TLSRootCAFile = "./test/certs/ca.pem" opts.TLSClientAuthPolicy = "require-verify" tcpAddr, _, nsqd := mustStartNSQD(opts) defer os.RemoveAll(opts.DataPath) defer nsqd.Exit() // with no cert conn, err := mustConnectNSQD(tcpAddr) test.Nil(t, err) defer conn.Close() data := identify(t, conn, map[string]interface{}{ "tls_v1": true, }, frameTypeResponse) r := struct { TLSv1 bool `json:"tls_v1"` }{} err = json.Unmarshal(data, &r) test.Nil(t, err) test.Equal(t, true, r.TLSv1) tlsConfig := &tls.Config{ InsecureSkipVerify: true, } tlsConn := tls.Client(conn, tlsConfig) _, err = nsq.ReadResponse(tlsConn) test.NotNil(t, err) // with invalid cert conn, err = mustConnectNSQD(tcpAddr) test.Nil(t, err) defer conn.Close() data = identify(t, conn, map[string]interface{}{ "tls_v1": true, }, frameTypeResponse) r = struct { TLSv1 bool `json:"tls_v1"` }{} err = json.Unmarshal(data, &r) test.Nil(t, err) test.Equal(t, true, r.TLSv1) cert, err := tls.LoadX509KeyPair("./test/certs/cert.pem", "./test/certs/key.pem") test.Nil(t, err) tlsConfig = &tls.Config{ Certificates: []tls.Certificate{cert}, InsecureSkipVerify: true, } tlsConn = tls.Client(conn, tlsConfig) _, err = nsq.ReadResponse(tlsConn) test.NotNil(t, err) // with valid cert conn, err = mustConnectNSQD(tcpAddr) test.Nil(t, err) defer conn.Close() data = identify(t, conn, map[string]interface{}{ "tls_v1": true, }, frameTypeResponse) r = struct { TLSv1 bool `json:"tls_v1"` }{} err = json.Unmarshal(data, &r) test.Nil(t, err) test.Equal(t, true, r.TLSv1) cert, err = tls.LoadX509KeyPair("./test/certs/client.pem", "./test/certs/client.key") test.Nil(t, err) tlsConfig = &tls.Config{ Certificates: []tls.Certificate{cert}, InsecureSkipVerify: true, } tlsConn = tls.Client(conn, tlsConfig) err = tlsConn.Handshake() test.Nil(t, err) resp, _ := nsq.ReadResponse(tlsConn) frameType, data, _ := nsq.UnpackResponse(resp) t.Logf("frameType: %d, data: %s", frameType, data) test.Equal(t, frameTypeResponse, frameType) test.Equal(t, []byte("OK"), data) } func TestDeflate(t *testing.T) { opts := NewOptions() opts.Logger = test.NewTestLogger(t) opts.LogLevel = LOG_DEBUG opts.DeflateEnabled = true tcpAddr, _, nsqd := mustStartNSQD(opts) defer os.RemoveAll(opts.DataPath) defer nsqd.Exit() conn, err := mustConnectNSQD(tcpAddr) test.Nil(t, err) defer conn.Close() data := identify(t, conn, map[string]interface{}{ "deflate": true, }, frameTypeResponse) r := struct { Deflate bool `json:"deflate"` }{} err = json.Unmarshal(data, &r) test.Nil(t, err) test.Equal(t, true, r.Deflate) compressConn := flate.NewReader(conn) resp, _ := nsq.ReadResponse(compressConn) frameType, data, _ := nsq.UnpackResponse(resp) t.Logf("frameType: %d, data: %s", frameType, data) test.Equal(t, frameTypeResponse, frameType) test.Equal(t, []byte("OK"), data) } type readWriter struct { io.Reader io.Writer } func TestSnappy(t *testing.T) { opts := NewOptions() opts.Logger = test.NewTestLogger(t) opts.LogLevel = LOG_DEBUG opts.SnappyEnabled = true tcpAddr, _, nsqd := mustStartNSQD(opts) defer os.RemoveAll(opts.DataPath) defer nsqd.Exit() conn, err := mustConnectNSQD(tcpAddr) test.Nil(t, err) defer conn.Close() data := identify(t, conn, map[string]interface{}{ "snappy": true, }, frameTypeResponse) r := struct { Snappy bool `json:"snappy"` }{} err = json.Unmarshal(data, &r) test.Nil(t, err) test.Equal(t, true, r.Snappy) compressConn := snappy.NewReader(conn) resp, _ := nsq.ReadResponse(compressConn) frameType, data, _ := nsq.UnpackResponse(resp) t.Logf("frameType: %d, data: %s", frameType, data) test.Equal(t, frameTypeResponse, frameType) test.Equal(t, []byte("OK"), data) msgBody := make([]byte, 128000) //lint:ignore SA1019 NewWriter is deprecated by NewBufferedWriter, but we don't want to buffer w := snappy.NewWriter(conn) rw := readWriter{compressConn, w} topicName := "test_snappy" + strconv.Itoa(int(time.Now().Unix())) sub(t, rw, topicName, "ch") _, err = nsq.Ready(1).WriteTo(rw) test.Nil(t, err) topic := nsqd.GetTopic(topicName) msg := NewMessage(topic.GenerateID(), msgBody) topic.PutMessage(msg) resp, _ = nsq.ReadResponse(compressConn) frameType, data, _ = nsq.UnpackResponse(resp) msgOut, _ := decodeMessage(data) test.Equal(t, frameTypeMessage, frameType) test.Equal(t, msg.ID, msgOut.ID) test.Equal(t, msg.Body, msgOut.Body) } func TestTLSDeflate(t *testing.T) { opts := NewOptions() opts.Logger = test.NewTestLogger(t) opts.LogLevel = LOG_DEBUG opts.DeflateEnabled = true opts.TLSCert = "./test/certs/cert.pem" opts.TLSKey = "./test/certs/key.pem" tcpAddr, _, nsqd := mustStartNSQD(opts) defer os.RemoveAll(opts.DataPath) defer nsqd.Exit() conn, err := mustConnectNSQD(tcpAddr) test.Nil(t, err) defer conn.Close() data := identify(t, conn, map[string]interface{}{ "tls_v1": true, "deflate": true, }, frameTypeResponse) r := struct { TLSv1 bool `json:"tls_v1"` Deflate bool `json:"deflate"` }{} err = json.Unmarshal(data, &r) test.Nil(t, err) test.Equal(t, true, r.TLSv1) test.Equal(t, true, r.Deflate) tlsConfig := &tls.Config{ InsecureSkipVerify: true, } tlsConn := tls.Client(conn, tlsConfig) err = tlsConn.Handshake() test.Nil(t, err) resp, _ := nsq.ReadResponse(tlsConn) frameType, data, _ := nsq.UnpackResponse(resp) t.Logf("frameType: %d, data: %s", frameType, data) test.Equal(t, frameTypeResponse, frameType) test.Equal(t, []byte("OK"), data) compressConn := flate.NewReader(tlsConn) resp, _ = nsq.ReadResponse(compressConn) frameType, data, _ = nsq.UnpackResponse(resp) t.Logf("frameType: %d, data: %s", frameType, data) test.Equal(t, frameTypeResponse, frameType) test.Equal(t, []byte("OK"), data) } func TestSampling(t *testing.T) { rand.Seed(time.Now().UTC().UnixNano()) num := 10000 sampleRate := 42 slack := 5 opts := NewOptions() opts.Logger = test.NewTestLogger(t) opts.LogLevel = LOG_DEBUG opts.MaxRdyCount = int64(num) tcpAddr, _, nsqd := mustStartNSQD(opts) defer os.RemoveAll(opts.DataPath) defer nsqd.Exit() conn, err := mustConnectNSQD(tcpAddr) test.Nil(t, err) defer conn.Close() data := identify(t, conn, map[string]interface{}{ "sample_rate": int32(sampleRate), }, frameTypeResponse) r := struct { SampleRate int32 `json:"sample_rate"` }{} err = json.Unmarshal(data, &r) test.Nil(t, err) test.Equal(t, int32(sampleRate), r.SampleRate) topicName := "test_sampling" + strconv.Itoa(int(time.Now().Unix())) topic := nsqd.GetTopic(topicName) for i := 0; i < num; i++ { msg := NewMessage(topic.GenerateID(), []byte("test body")) topic.PutMessage(msg) } channel := topic.GetChannel("ch") // let the topic drain into the channel time.Sleep(50 * time.Millisecond) sub(t, conn, topicName, "ch") _, err = nsq.Ready(num).WriteTo(conn) test.Nil(t, err) go func() { for { _, err := nsq.ReadResponse(conn) if err != nil { return } } }() doneChan := make(chan int) go func() { for { if channel.Depth() == 0 { close(doneChan) return } time.Sleep(5 * time.Millisecond) } }() <-doneChan channel.inFlightMutex.Lock() numInFlight := len(channel.inFlightMessages) channel.inFlightMutex.Unlock() test.Equal(t, true, numInFlight <= int(float64(num)*float64(sampleRate+slack)/100.0)) test.Equal(t, true, numInFlight >= int(float64(num)*float64(sampleRate-slack)/100.0)) } func TestTLSSnappy(t *testing.T) { opts := NewOptions() opts.Logger = test.NewTestLogger(t) opts.LogLevel = LOG_DEBUG opts.SnappyEnabled = true opts.TLSCert = "./test/certs/cert.pem" opts.TLSKey = "./test/certs/key.pem" tcpAddr, _, nsqd := mustStartNSQD(opts) defer os.RemoveAll(opts.DataPath) defer nsqd.Exit() conn, err := mustConnectNSQD(tcpAddr) test.Nil(t, err) defer conn.Close() data := identify(t, conn, map[string]interface{}{ "tls_v1": true, "snappy": true, }, frameTypeResponse) r := struct { TLSv1 bool `json:"tls_v1"` Snappy bool `json:"snappy"` }{} err = json.Unmarshal(data, &r) test.Nil(t, err) test.Equal(t, true, r.TLSv1) test.Equal(t, true, r.Snappy) tlsConfig := &tls.Config{ InsecureSkipVerify: true, } tlsConn := tls.Client(conn, tlsConfig) err = tlsConn.Handshake() test.Nil(t, err) resp, _ := nsq.ReadResponse(tlsConn) frameType, data, _ := nsq.UnpackResponse(resp) t.Logf("frameType: %d, data: %s", frameType, data) test.Equal(t, frameTypeResponse, frameType) test.Equal(t, []byte("OK"), data) compressConn := snappy.NewReader(tlsConn) resp, _ = nsq.ReadResponse(compressConn) frameType, data, _ = nsq.UnpackResponse(resp) t.Logf("frameType: %d, data: %s", frameType, data) test.Equal(t, frameTypeResponse, frameType) test.Equal(t, []byte("OK"), data) } func TestClientMsgTimeout(t *testing.T) { opts := NewOptions() opts.Logger = test.NewTestLogger(t) opts.LogLevel = LOG_DEBUG opts.QueueScanRefreshInterval = 100 * time.Millisecond tcpAddr, _, nsqd := mustStartNSQD(opts) defer os.RemoveAll(opts.DataPath) defer nsqd.Exit() topicName := "test_cmsg_timeout" + strconv.Itoa(int(time.Now().Unix())) topic := nsqd.GetTopic(topicName) ch := topic.GetChannel("ch") msg := NewMessage(topic.GenerateID(), make([]byte, 100)) topic.PutMessage(msg) // without this the race detector thinks there's a write // to msg.Attempts that races with the read in the protocol's messagePump... // it does not reflect a realistically possible condition topic.PutMessage(NewMessage(topic.GenerateID(), make([]byte, 100))) conn, err := mustConnectNSQD(tcpAddr) test.Nil(t, err) defer conn.Close() identify(t, conn, map[string]interface{}{ "msg_timeout": 1000, }, frameTypeResponse) sub(t, conn, topicName, "ch") test.Equal(t, 0, int(atomic.LoadUint64(&ch.timeoutCount))) test.Equal(t, 0, int(atomic.LoadUint64(&ch.requeueCount))) _, err = nsq.Ready(1).WriteTo(conn) test.Nil(t, err) resp, _ := nsq.ReadResponse(conn) _, data, _ := nsq.UnpackResponse(resp) msgOut, _ := decodeMessage(data) test.Equal(t, msg.ID, msgOut.ID) test.Equal(t, msg.Body, msgOut.Body) _, err = nsq.Ready(0).WriteTo(conn) test.Nil(t, err) time.Sleep(1150 * time.Millisecond) test.Equal(t, 1, int(atomic.LoadUint64(&ch.timeoutCount))) test.Equal(t, 0, int(atomic.LoadUint64(&ch.requeueCount))) _, err = nsq.Finish(nsq.MessageID(msgOut.ID)).WriteTo(conn) test.Nil(t, err) resp, _ = nsq.ReadResponse(conn) frameType, data, _ := nsq.UnpackResponse(resp) test.Equal(t, frameTypeError, frameType) test.Equal(t, fmt.Sprintf("E_FIN_FAILED FIN %s failed ID not in flight", msgOut.ID), string(data)) } func TestBadFin(t *testing.T) { opts := NewOptions() opts.Logger = test.NewTestLogger(t) opts.LogLevel = LOG_DEBUG tcpAddr, _, nsqd := mustStartNSQD(opts) defer os.RemoveAll(opts.DataPath) defer nsqd.Exit() conn, err := mustConnectNSQD(tcpAddr) test.Nil(t, err) defer conn.Close() identify(t, conn, map[string]interface{}{}, frameTypeResponse) sub(t, conn, "test_fin", "ch") fin := nsq.Finish(nsq.MessageID{}) fin.Params[0] = []byte("") _, err = fin.WriteTo(conn) test.Nil(t, err) resp, _ := nsq.ReadResponse(conn) frameType, data, _ := nsq.UnpackResponse(resp) test.Equal(t, frameTypeError, frameType) test.Equal(t, "E_INVALID invalid message ID", string(data)) } func TestReqTimeoutRange(t *testing.T) { opts := NewOptions() opts.Logger = test.NewTestLogger(t) opts.LogLevel = LOG_DEBUG opts.MaxReqTimeout = 1 * time.Minute tcpAddr, _, nsqd := mustStartNSQD(opts) defer os.RemoveAll(opts.DataPath) defer nsqd.Exit() topicName := "test_req" + strconv.Itoa(int(time.Now().Unix())) conn, err := mustConnectNSQD(tcpAddr) test.Nil(t, err) defer conn.Close() identify(t, conn, nil, frameTypeResponse) sub(t, conn, topicName, "ch") topic := nsqd.GetTopic(topicName) channel := topic.GetChannel("ch") msg := NewMessage(topic.GenerateID(), []byte("test body")) topic.PutMessage(msg) _, err = nsq.Ready(1).WriteTo(conn) test.Nil(t, err) resp, err := nsq.ReadResponse(conn) test.Nil(t, err) frameType, data, _ := nsq.UnpackResponse(resp) msgOut, _ := decodeMessage(data) test.Equal(t, frameTypeMessage, frameType) test.Equal(t, msg.ID, msgOut.ID) _, err = nsq.Requeue(nsq.MessageID(msg.ID), -1).WriteTo(conn) test.Nil(t, err) // It should be immediately available for another attempt resp, err = nsq.ReadResponse(conn) test.Nil(t, err) frameType, data, _ = nsq.UnpackResponse(resp) msgOut, _ = decodeMessage(data) test.Equal(t, frameTypeMessage, frameType) test.Equal(t, msg.ID, msgOut.ID) // The priority (processing time) should be >= this minTs := time.Now().Add(opts.MaxReqTimeout).UnixNano() _, err = nsq.Requeue(nsq.MessageID(msg.ID), opts.MaxReqTimeout*2).WriteTo(conn) test.Nil(t, err) time.Sleep(100 * time.Millisecond) channel.deferredMutex.Lock() pqItem := channel.deferredMessages[msg.ID] channel.deferredMutex.Unlock() test.NotNil(t, pqItem) test.Equal(t, true, pqItem.Priority >= minTs) } func TestClientAuth(t *testing.T) { authResponse := `{"ttl":1, "authorizations":[]}` authSecret := "testsecret" authError := "E_UNAUTHORIZED AUTH no authorizations found" authSuccess := "" tlsEnabled := false commonName := "" httpAuthRequestMethod := "get" runAuthTest(t, authResponse, authSecret, authError, authSuccess, tlsEnabled, commonName, httpAuthRequestMethod) // now one that will succeed authResponse = `{"ttl":10, "authorizations": [{"topic":"test", "channels":[".*"], "permissions":["subscribe","publish"]}] }` authError = "" authSuccess = `{"identity":"","identity_url":"","permission_count":1}` runAuthTest(t, authResponse, authSecret, authError, authSuccess, tlsEnabled, commonName, httpAuthRequestMethod) // one with TLS enabled tlsEnabled = true commonName = "test.local" runAuthTest(t, authResponse, authSecret, authError, authSuccess, tlsEnabled, commonName, httpAuthRequestMethod) // test POST based authentication httpAuthRequestMethod = "post" runAuthTest(t, authResponse, authSecret, authError, authSuccess, tlsEnabled, commonName, httpAuthRequestMethod) } func runAuthTest(t *testing.T, authResponse string, authSecret string, authError string, authSuccess string, tlsEnabled bool, commonName string, httpAuthRequestMethod string) { var err error var expectedRemoteIP string expectedTLS := "false" if tlsEnabled { expectedTLS = "true" } authd := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { t.Logf("in test auth handler %s", r.RequestURI) test.Equal(t, httpAuthRequestMethod, strings.ToLower(r.Method)) var values url.Values if r.Method == "POST" { err = json.NewDecoder(r.Body).Decode(&values) if err != nil { t.Error(err) } } else { r.ParseForm() values = r.Form } test.Equal(t, expectedRemoteIP, values.Get("remote_ip")) test.Equal(t, expectedTLS, values.Get("tls")) test.Equal(t, commonName, values.Get("common_name")) test.Equal(t, authSecret, values.Get("secret")) fmt.Fprint(w, authResponse) })) defer authd.Close() addr, err := url.Parse(authd.URL) test.Nil(t, err) opts := NewOptions() opts.Logger = test.NewTestLogger(t) opts.LogLevel = LOG_DEBUG opts.AuthHTTPAddresses = []string{addr.Host} opts.AuthHTTPRequestMethod = httpAuthRequestMethod if tlsEnabled { opts.TLSCert = "./test/certs/server.pem" opts.TLSKey = "./test/certs/server.key" opts.TLSClientAuthPolicy = "require" } tcpAddr, _, nsqd := mustStartNSQD(opts) defer os.RemoveAll(opts.DataPath) defer nsqd.Exit() conn, err := mustConnectNSQD(tcpAddr) test.Nil(t, err) defer conn.Close() data := identify(t, conn, map[string]interface{}{ "tls_v1": tlsEnabled, }, frameTypeResponse) r := struct { TLSv1 bool `json:"tls_v1"` }{} err = json.Unmarshal(data, &r) test.Nil(t, err) test.Equal(t, tlsEnabled, r.TLSv1) var c io.ReadWriter var tlsConn *tls.Conn c = conn if tlsEnabled { cert, err := tls.LoadX509KeyPair("./test/certs/cert.pem", "./test/certs/key.pem") test.Nil(t, err) tlsConfig := &tls.Config{ Certificates: []tls.Certificate{cert}, InsecureSkipVerify: true, } tlsConn = tls.Client(conn, tlsConfig) err = tlsConn.Handshake() test.Nil(t, err) c = tlsConn resp, _ := nsq.ReadResponse(tlsConn) frameType, data, _ := nsq.UnpackResponse(resp) t.Logf("frameType: %d, data: %s", frameType, data) test.Equal(t, frameTypeResponse, frameType) test.Equal(t, []byte("OK"), data) } expectedRemoteIP, _, _ = net.SplitHostPort(conn.LocalAddr().String()) authCmd(t, c, authSecret, authSuccess) if authError != "" { readValidate(t, c, frameTypeError, authError) } else { sub(t, c, "test", "ch") } } func TestIOLoopReturnsClientErrWhenSendFails(t *testing.T) { fakeConn := test.NewFakeNetConn() fakeConn.WriteFunc = func(b []byte) (int, error) { return 0, errors.New("write error") } testIOLoopReturnsClientErr(t, fakeConn) } func TestIOLoopReturnsClientErrWhenSendSucceeds(t *testing.T) { fakeConn := test.NewFakeNetConn() fakeConn.WriteFunc = func(b []byte) (int, error) { return len(b), nil } testIOLoopReturnsClientErr(t, fakeConn) } func testIOLoopReturnsClientErr(t *testing.T, fakeConn test.FakeNetConn) { fakeConn.ReadFunc = func(b []byte) (int, error) { return copy(b, []byte("INVALID_COMMAND\n")), nil } opts := NewOptions() opts.Logger = test.NewTestLogger(t) opts.LogLevel = LOG_DEBUG nsqd, err := New(opts) test.Nil(t, err) prot := &protocolV2{nsqd: nsqd} defer prot.nsqd.Exit() client := prot.NewClient(fakeConn) err = prot.IOLoop(client) test.NotNil(t, err) test.Equal(t, "E_INVALID invalid command INVALID_COMMAND", err.Error()) test.NotNil(t, err.(*protocol.FatalClientErr)) } func BenchmarkProtocolV2Exec(b *testing.B) { b.StopTimer() opts := NewOptions() opts.Logger = test.NewTestLogger(b) nsqd, _ := New(opts) p := &protocolV2{nsqd} c := newClientV2(0, nil, nsqd) params := [][]byte{[]byte("NOP")} b.StartTimer() for i := 0; i < b.N; i++ { p.Exec(c, params) } } func benchmarkProtocolV2PubMultiTopic(b *testing.B, numTopics int) { var wg sync.WaitGroup b.StopTimer() opts := NewOptions() size := 200 batchSize := int(opts.MaxBodySize) / (size + 4) opts.Logger = test.NewTestLogger(b) opts.MemQueueSize = int64(b.N) tcpAddr, _, nsqd := mustStartNSQD(opts) defer os.RemoveAll(opts.DataPath) msg := make([]byte, size) batch := make([][]byte, batchSize) for i := range batch { batch[i] = msg } b.SetBytes(int64(len(msg))) b.StartTimer() for j := 0; j < numTopics; j++ { topicName := fmt.Sprintf("bench_v2_pub_multi_topic_%d_%d", j, time.Now().Unix()) wg.Add(1) go func() { conn, err := mustConnectNSQD(tcpAddr) if err != nil { panic(err.Error()) } rw := bufio.NewReadWriter(bufio.NewReader(conn), bufio.NewWriter(conn)) num := b.N / numTopics / batchSize wg.Add(1) go func() { for i := 0; i < num; i++ { cmd, _ := nsq.MultiPublish(topicName, batch) _, err := cmd.WriteTo(rw) if err != nil { panic(err.Error()) } err = rw.Flush() if err != nil { panic(err.Error()) } } wg.Done() }() wg.Add(1) go func() { for i := 0; i < num; i++ { resp, err := nsq.ReadResponse(rw) if err != nil { panic(err.Error()) } _, data, _ := nsq.UnpackResponse(resp) if !bytes.Equal(data, []byte("OK")) { panic("invalid response") } } wg.Done() }() wg.Done() }() } wg.Wait() b.StopTimer() nsqd.Exit() } func BenchmarkProtocolV2PubMultiTopic1(b *testing.B) { benchmarkProtocolV2PubMultiTopic(b, 1) } func BenchmarkProtocolV2PubMultiTopic2(b *testing.B) { benchmarkProtocolV2PubMultiTopic(b, 2) } func BenchmarkProtocolV2PubMultiTopic4(b *testing.B) { benchmarkProtocolV2PubMultiTopic(b, 4) } func BenchmarkProtocolV2PubMultiTopic8(b *testing.B) { benchmarkProtocolV2PubMultiTopic(b, 8) } func BenchmarkProtocolV2PubMultiTopic16(b *testing.B) { benchmarkProtocolV2PubMultiTopic(b, 16) } func BenchmarkProtocolV2PubMultiTopic32(b *testing.B) { benchmarkProtocolV2PubMultiTopic(b, 32) } func benchmarkProtocolV2Pub(b *testing.B, size int) { var wg sync.WaitGroup b.StopTimer() opts := NewOptions() batchSize := int(opts.MaxBodySize) / (size + 4) opts.Logger = test.NewTestLogger(b) opts.MemQueueSize = int64(b.N) tcpAddr, _, nsqd := mustStartNSQD(opts) defer os.RemoveAll(opts.DataPath) msg := make([]byte, size) batch := make([][]byte, batchSize) for i := range batch { batch[i] = msg } topicName := "bench_v2_pub" + strconv.Itoa(int(time.Now().Unix())) b.SetBytes(int64(len(msg))) b.StartTimer() for j := 0; j < runtime.GOMAXPROCS(0); j++ { wg.Add(1) go func() { conn, err := mustConnectNSQD(tcpAddr) if err != nil { panic(err.Error()) } rw := bufio.NewReadWriter(bufio.NewReader(conn), bufio.NewWriter(conn)) num := b.N / runtime.GOMAXPROCS(0) / batchSize wg.Add(1) go func() { for i := 0; i < num; i++ { cmd, _ := nsq.MultiPublish(topicName, batch) _, err := cmd.WriteTo(rw) if err != nil { panic(err.Error()) } err = rw.Flush() if err != nil { panic(err.Error()) } } wg.Done() }() wg.Add(1) go func() { for i := 0; i < num; i++ { resp, err := nsq.ReadResponse(rw) if err != nil { panic(err.Error()) } _, data, _ := nsq.UnpackResponse(resp) if !bytes.Equal(data, []byte("OK")) { panic("invalid response") } } wg.Done() }() wg.Done() }() } wg.Wait() b.StopTimer() nsqd.Exit() } func BenchmarkProtocolV2Pub256(b *testing.B) { benchmarkProtocolV2Pub(b, 256) } func BenchmarkProtocolV2Pub512(b *testing.B) { benchmarkProtocolV2Pub(b, 512) } func BenchmarkProtocolV2Pub1k(b *testing.B) { benchmarkProtocolV2Pub(b, 1024) } func BenchmarkProtocolV2Pub2k(b *testing.B) { benchmarkProtocolV2Pub(b, 2*1024) } func BenchmarkProtocolV2Pub4k(b *testing.B) { benchmarkProtocolV2Pub(b, 4*1024) } func BenchmarkProtocolV2Pub8k(b *testing.B) { benchmarkProtocolV2Pub(b, 8*1024) } func BenchmarkProtocolV2Pub16k(b *testing.B) { benchmarkProtocolV2Pub(b, 16*1024) } func BenchmarkProtocolV2Pub32k(b *testing.B) { benchmarkProtocolV2Pub(b, 32*1024) } func BenchmarkProtocolV2Pub64k(b *testing.B) { benchmarkProtocolV2Pub(b, 64*1024) } func BenchmarkProtocolV2Pub128k(b *testing.B) { benchmarkProtocolV2Pub(b, 128*1024) } func BenchmarkProtocolV2Pub256k(b *testing.B) { benchmarkProtocolV2Pub(b, 256*1024) } func BenchmarkProtocolV2Pub512k(b *testing.B) { benchmarkProtocolV2Pub(b, 512*1024) } func BenchmarkProtocolV2Pub1m(b *testing.B) { benchmarkProtocolV2Pub(b, 1024*1024) } func benchmarkProtocolV2Sub(b *testing.B, size int) { var wg sync.WaitGroup b.StopTimer() opts := NewOptions() opts.Logger = test.NewTestLogger(b) opts.MemQueueSize = int64(b.N) tcpAddr, _, nsqd := mustStartNSQD(opts) defer os.RemoveAll(opts.DataPath) msg := make([]byte, size) topicName := "bench_v2_sub" + strconv.Itoa(b.N) + strconv.Itoa(int(time.Now().Unix())) topic := nsqd.GetTopic(topicName) for i := 0; i < b.N; i++ { msg := NewMessage(topic.GenerateID(), msg) topic.PutMessage(msg) } topic.GetChannel("ch") b.SetBytes(int64(len(msg))) goChan := make(chan int) rdyChan := make(chan int) workers := runtime.GOMAXPROCS(0) for j := 0; j < workers; j++ { wg.Add(1) go func() { subWorker(b.N, workers, tcpAddr, topicName, rdyChan, goChan) wg.Done() }() <-rdyChan } b.StartTimer() close(goChan) wg.Wait() b.StopTimer() nsqd.Exit() } func subWorker(n int, workers int, tcpAddr net.Addr, topicName string, rdyChan chan int, goChan chan int) { conn, err := mustConnectNSQD(tcpAddr) if err != nil { panic(err.Error()) } rw := bufio.NewReadWriter(bufio.NewReader(conn), bufio.NewWriterSize(conn, 65536)) identify(nil, conn, nil, frameTypeResponse) sub(nil, conn, topicName, "ch") rdyCount := int(math.Min(math.Max(float64(n/workers), 1), 2500)) rdyChan <- 1 <-goChan nsq.Ready(rdyCount).WriteTo(rw) rw.Flush() num := n / workers for i := 0; i < num; i++ { resp, err := nsq.ReadResponse(rw) if err != nil { panic(err.Error()) } frameType, data, err := nsq.UnpackResponse(resp) if err != nil { panic(err.Error()) } if frameType != frameTypeMessage { panic("got something else") } msg, err := decodeMessage(data) if err != nil { panic(err.Error()) } nsq.Finish(nsq.MessageID(msg.ID)).WriteTo(rw) if (i+1)%rdyCount == 0 || i+1 == num { if i+1 == num { nsq.Ready(0).WriteTo(conn) } rw.Flush() } } } func BenchmarkProtocolV2Sub256(b *testing.B) { benchmarkProtocolV2Sub(b, 256) } func BenchmarkProtocolV2Sub512(b *testing.B) { benchmarkProtocolV2Sub(b, 512) } func BenchmarkProtocolV2Sub1k(b *testing.B) { benchmarkProtocolV2Sub(b, 1024) } func BenchmarkProtocolV2Sub2k(b *testing.B) { benchmarkProtocolV2Sub(b, 2*1024) } func BenchmarkProtocolV2Sub4k(b *testing.B) { benchmarkProtocolV2Sub(b, 4*1024) } func BenchmarkProtocolV2Sub8k(b *testing.B) { benchmarkProtocolV2Sub(b, 8*1024) } func BenchmarkProtocolV2Sub16k(b *testing.B) { benchmarkProtocolV2Sub(b, 16*1024) } func BenchmarkProtocolV2Sub32k(b *testing.B) { benchmarkProtocolV2Sub(b, 32*1024) } func BenchmarkProtocolV2Sub64k(b *testing.B) { benchmarkProtocolV2Sub(b, 64*1024) } func BenchmarkProtocolV2Sub128k(b *testing.B) { benchmarkProtocolV2Sub(b, 128*1024) } func BenchmarkProtocolV2Sub256k(b *testing.B) { benchmarkProtocolV2Sub(b, 256*1024) } func BenchmarkProtocolV2Sub512k(b *testing.B) { benchmarkProtocolV2Sub(b, 512*1024) } func BenchmarkProtocolV2Sub1m(b *testing.B) { benchmarkProtocolV2Sub(b, 1024*1024) } func benchmarkProtocolV2MultiSub(b *testing.B, num int) { var wg sync.WaitGroup b.StopTimer() opts := NewOptions() opts.Logger = test.NewTestLogger(b) opts.MemQueueSize = int64(b.N) tcpAddr, _, nsqd := mustStartNSQD(opts) defer os.RemoveAll(opts.DataPath) msg := make([]byte, 256) b.SetBytes(int64(len(msg) * num)) goChan := make(chan int) rdyChan := make(chan int) workers := runtime.GOMAXPROCS(0) for i := 0; i < num; i++ { topicName := "bench_v2" + strconv.Itoa(b.N) + "_" + strconv.Itoa(i) + "_" + strconv.Itoa(int(time.Now().Unix())) topic := nsqd.GetTopic(topicName) for i := 0; i < b.N; i++ { msg := NewMessage(topic.GenerateID(), msg) topic.PutMessage(msg) } topic.GetChannel("ch") for j := 0; j < workers; j++ { wg.Add(1) go func() { subWorker(b.N, workers, tcpAddr, topicName, rdyChan, goChan) wg.Done() }() <-rdyChan } } b.StartTimer() close(goChan) wg.Wait() b.StopTimer() nsqd.Exit() } func BenchmarkProtocolV2MultiSub1(b *testing.B) { benchmarkProtocolV2MultiSub(b, 1) } func BenchmarkProtocolV2MultiSub2(b *testing.B) { benchmarkProtocolV2MultiSub(b, 2) } func BenchmarkProtocolV2MultiSub4(b *testing.B) { benchmarkProtocolV2MultiSub(b, 4) } func BenchmarkProtocolV2MultiSub8(b *testing.B) { benchmarkProtocolV2MultiSub(b, 8) } func BenchmarkProtocolV2MultiSub16(b *testing.B) { benchmarkProtocolV2MultiSub(b, 16) } ================================================ FILE: nsqd/protocol_v2_unixsocket_test.go ================================================ package nsqd import ( "bufio" "bytes" "compress/flate" "crypto/tls" "encoding/json" "errors" "fmt" "math" "math/rand" "net" "os" "path" "runtime" "strconv" "sync" "sync/atomic" "testing" "time" "github.com/golang/snappy" "github.com/nsqio/go-nsq" "github.com/nsqio/nsq/internal/protocol" "github.com/nsqio/nsq/internal/test" ) func mustUnixSocketStartNSQD(opts *Options) (net.Addr, net.Addr, *NSQD) { tmpDir := os.TempDir() opts.TCPAddress = path.Join(tmpDir, fmt.Sprintf("nsqd-%d.sock", rand.Int())) opts.HTTPAddress = path.Join(tmpDir, fmt.Sprintf("nsqd-%d.sock", rand.Int())) if opts.DataPath == "" { tmpDir, err := os.MkdirTemp("", "nsq-test-") if err != nil { panic(err) } opts.DataPath = tmpDir } nsqd, err := New(opts) if err != nil { panic(err) } go func() { err := nsqd.Main() if err != nil { panic(err) } }() return nsqd.RealTCPAddr(), nsqd.RealHTTPAddr(), nsqd } func mustUnixSocketConnectNSQD(addr net.Addr) (net.Conn, error) { conn, err := net.DialTimeout("unix", addr.String(), time.Second) if err != nil { return nil, err } conn.Write(nsq.MagicV2) return conn, nil } // exercise the basic operations of the V2 protocol func TestUnixSocketBasicV2(t *testing.T) { opts := NewOptions() opts.Logger = test.NewTestLogger(t) opts.ClientTimeout = 60 * time.Second addr, _, nsqd := mustUnixSocketStartNSQD(opts) defer os.RemoveAll(opts.DataPath) defer nsqd.Exit() topicName := "test_v2" + strconv.Itoa(int(time.Now().Unix())) topic := nsqd.GetTopic(topicName) msg := NewMessage(topic.GenerateID(), []byte("test body")) topic.PutMessage(msg) conn, err := mustUnixSocketConnectNSQD(addr) test.Nil(t, err) defer conn.Close() identify(t, conn, nil, frameTypeResponse) sub(t, conn, topicName, "ch") _, err = nsq.Ready(1).WriteTo(conn) test.Nil(t, err) resp, err := nsq.ReadResponse(conn) test.Nil(t, err) frameType, data, _ := nsq.UnpackResponse(resp) msgOut, _ := decodeMessage(data) test.Equal(t, frameTypeMessage, frameType) test.Equal(t, msg.ID, msgOut.ID) test.Equal(t, msg.Body, msgOut.Body) test.Equal(t, uint16(1), msgOut.Attempts) } func TestUnixSocketMultipleConsumerV2(t *testing.T) { msgChan := make(chan *Message) opts := NewOptions() opts.Logger = test.NewTestLogger(t) opts.ClientTimeout = 60 * time.Second addr, _, nsqd := mustUnixSocketStartNSQD(opts) defer os.RemoveAll(opts.DataPath) defer nsqd.Exit() topicName := "test_multiple_v2" + strconv.Itoa(int(time.Now().Unix())) topic := nsqd.GetTopic(topicName) msg := NewMessage(topic.GenerateID(), []byte("test body")) topic.GetChannel("ch1") topic.GetChannel("ch2") topic.PutMessage(msg) for _, i := range []string{"1", "2"} { conn, err := mustUnixSocketConnectNSQD(addr) test.Nil(t, err) defer conn.Close() identify(t, conn, nil, frameTypeResponse) sub(t, conn, topicName, "ch"+i) _, err = nsq.Ready(1).WriteTo(conn) test.Nil(t, err) go func(c net.Conn) { resp, err := nsq.ReadResponse(c) test.Nil(t, err) _, data, err := nsq.UnpackResponse(resp) test.Nil(t, err) msg, err := decodeMessage(data) test.Nil(t, err) msgChan <- msg }(conn) } msgOut := <-msgChan test.Equal(t, msg.ID, msgOut.ID) test.Equal(t, msg.Body, msgOut.Body) test.Equal(t, uint16(1), msgOut.Attempts) msgOut = <-msgChan test.Equal(t, msg.ID, msgOut.ID) test.Equal(t, msg.Body, msgOut.Body) test.Equal(t, uint16(1), msgOut.Attempts) } func TestUnixSocketClientTimeout(t *testing.T) { topicName := "test_client_timeout_v2" + strconv.Itoa(int(time.Now().Unix())) opts := NewOptions() opts.Logger = test.NewTestLogger(t) opts.ClientTimeout = 150 * time.Millisecond opts.LogLevel = LOG_DEBUG addr, _, nsqd := mustUnixSocketStartNSQD(opts) defer os.RemoveAll(opts.DataPath) defer nsqd.Exit() conn, err := mustUnixSocketConnectNSQD(addr) test.Nil(t, err) defer conn.Close() identify(t, conn, nil, frameTypeResponse) sub(t, conn, topicName, "ch") time.Sleep(150 * time.Millisecond) // depending on timing there may be 1 or 2 hearbeats sent // just read until we get an error timer := time.After(100 * time.Millisecond) for { select { case <-timer: t.Fatalf("test timed out") default: _, err := nsq.ReadResponse(conn) if err != nil { goto done } } } done: } func TestUnixSocketClientHeartbeat(t *testing.T) { topicName := "test_hb_v2" + strconv.Itoa(int(time.Now().Unix())) opts := NewOptions() opts.Logger = test.NewTestLogger(t) opts.ClientTimeout = 200 * time.Millisecond addr, _, nsqd := mustUnixSocketStartNSQD(opts) defer os.RemoveAll(opts.DataPath) defer nsqd.Exit() conn, err := mustUnixSocketConnectNSQD(addr) test.Nil(t, err) defer conn.Close() identify(t, conn, nil, frameTypeResponse) sub(t, conn, topicName, "ch") _, err = nsq.Ready(1).WriteTo(conn) test.Nil(t, err) resp, _ := nsq.ReadResponse(conn) _, data, _ := nsq.UnpackResponse(resp) test.Equal(t, []byte("_heartbeat_"), data) time.Sleep(20 * time.Millisecond) _, err = nsq.Nop().WriteTo(conn) test.Nil(t, err) // wait long enough that would have timed out (had we not sent the above cmd) time.Sleep(100 * time.Millisecond) _, err = nsq.Nop().WriteTo(conn) test.Nil(t, err) } func TestUnixSocketClientHeartbeatDisableSUB(t *testing.T) { topicName := "test_hb_v2" + strconv.Itoa(int(time.Now().Unix())) opts := NewOptions() opts.Logger = test.NewTestLogger(t) opts.ClientTimeout = 200 * time.Millisecond opts.LogLevel = LOG_DEBUG addr, _, nsqd := mustUnixSocketStartNSQD(opts) defer os.RemoveAll(opts.DataPath) defer nsqd.Exit() conn, err := mustUnixSocketConnectNSQD(addr) test.Nil(t, err) defer conn.Close() identify(t, conn, map[string]interface{}{ "heartbeat_interval": -1, }, frameTypeResponse) subFail(t, conn, topicName, "ch") } func TestUnixSocketClientHeartbeatDisable(t *testing.T) { opts := NewOptions() opts.Logger = test.NewTestLogger(t) opts.ClientTimeout = 100 * time.Millisecond addr, _, nsqd := mustUnixSocketStartNSQD(opts) defer os.RemoveAll(opts.DataPath) defer nsqd.Exit() conn, err := mustUnixSocketConnectNSQD(addr) test.Nil(t, err) defer conn.Close() identify(t, conn, map[string]interface{}{ "heartbeat_interval": -1, }, frameTypeResponse) time.Sleep(150 * time.Millisecond) _, err = nsq.Nop().WriteTo(conn) test.Nil(t, err) } func TestUnixSocketMaxHeartbeatIntervalValid(t *testing.T) { opts := NewOptions() opts.Logger = test.NewTestLogger(t) opts.MaxHeartbeatInterval = 300 * time.Second addr, _, nsqd := mustUnixSocketStartNSQD(opts) defer os.RemoveAll(opts.DataPath) defer nsqd.Exit() conn, err := mustUnixSocketConnectNSQD(addr) test.Nil(t, err) defer conn.Close() hbi := int(opts.MaxHeartbeatInterval / time.Millisecond) identify(t, conn, map[string]interface{}{ "heartbeat_interval": hbi, }, frameTypeResponse) } func TestUnixSocketMaxHeartbeatIntervalInvalid(t *testing.T) { opts := NewOptions() opts.Logger = test.NewTestLogger(t) opts.MaxHeartbeatInterval = 300 * time.Second addr, _, nsqd := mustUnixSocketStartNSQD(opts) defer os.RemoveAll(opts.DataPath) defer nsqd.Exit() conn, err := mustUnixSocketConnectNSQD(addr) test.Nil(t, err) defer conn.Close() hbi := int(opts.MaxHeartbeatInterval/time.Millisecond + 1) data := identify(t, conn, map[string]interface{}{ "heartbeat_interval": hbi, }, frameTypeError) test.Equal(t, "E_BAD_BODY IDENTIFY heartbeat interval (300001) is invalid", string(data)) } func TestUnixSocketPausing(t *testing.T) { topicName := "test_pause_v2" + strconv.Itoa(int(time.Now().Unix())) opts := NewOptions() opts.Logger = test.NewTestLogger(t) addr, _, nsqd := mustUnixSocketStartNSQD(opts) defer os.RemoveAll(opts.DataPath) defer nsqd.Exit() conn, err := mustUnixSocketConnectNSQD(addr) test.Nil(t, err) defer conn.Close() identify(t, conn, nil, frameTypeResponse) sub(t, conn, topicName, "ch") _, err = nsq.Ready(1).WriteTo(conn) test.Nil(t, err) topic := nsqd.GetTopic(topicName) msg := NewMessage(topic.GenerateID(), []byte("test body")) channel := topic.GetChannel("ch") topic.PutMessage(msg) // receive the first message via the client, finish it, and send new RDY resp, _ := nsq.ReadResponse(conn) _, data, _ := nsq.UnpackResponse(resp) msg, _ = decodeMessage(data) test.Equal(t, []byte("test body"), msg.Body) _, err = nsq.Finish(nsq.MessageID(msg.ID)).WriteTo(conn) test.Nil(t, err) _, err = nsq.Ready(1).WriteTo(conn) test.Nil(t, err) // sleep to allow the RDY state to take effect time.Sleep(50 * time.Millisecond) // pause the channel... the client shouldn't receive any more messages channel.Pause() // sleep to allow the paused state to take effect time.Sleep(50 * time.Millisecond) msg = NewMessage(topic.GenerateID(), []byte("test body2")) topic.PutMessage(msg) // allow the client to possibly get a message, the test would hang indefinitely // if pausing was not working time.Sleep(50 * time.Millisecond) msg = <-channel.memoryMsgChan test.Equal(t, []byte("test body2"), msg.Body) // unpause the channel... the client should now be pushed a message channel.UnPause() msg = NewMessage(topic.GenerateID(), []byte("test body3")) topic.PutMessage(msg) resp, _ = nsq.ReadResponse(conn) _, data, _ = nsq.UnpackResponse(resp) msg, _ = decodeMessage(data) test.Equal(t, []byte("test body3"), msg.Body) } func TestUnixSocketEmptyCommand(t *testing.T) { opts := NewOptions() opts.Logger = test.NewTestLogger(t) addr, _, nsqd := mustUnixSocketStartNSQD(opts) defer os.RemoveAll(opts.DataPath) defer nsqd.Exit() conn, err := mustUnixSocketConnectNSQD(addr) test.Nil(t, err) defer conn.Close() _, err = conn.Write([]byte("\n\n")) test.Nil(t, err) // if we didn't panic here we're good, see issue #120 } func TestUnixSocketSizeLimits(t *testing.T) { opts := NewOptions() opts.Logger = test.NewTestLogger(t) opts.LogLevel = LOG_DEBUG opts.MaxMsgSize = 100 opts.MaxBodySize = 1000 addr, _, nsqd := mustUnixSocketStartNSQD(opts) defer os.RemoveAll(opts.DataPath) defer nsqd.Exit() conn, err := mustUnixSocketConnectNSQD(addr) test.Nil(t, err) defer conn.Close() topicName := "test_limits_v2" + strconv.Itoa(int(time.Now().Unix())) identify(t, conn, nil, frameTypeResponse) sub(t, conn, topicName, "ch") // PUB that's valid nsq.Publish(topicName, make([]byte, 95)).WriteTo(conn) resp, _ := nsq.ReadResponse(conn) frameType, data, _ := nsq.UnpackResponse(resp) t.Logf("frameType: %d, data: %s", frameType, data) test.Equal(t, frameTypeResponse, frameType) test.Equal(t, []byte("OK"), data) // PUB that's invalid (too big) nsq.Publish(topicName, make([]byte, 105)).WriteTo(conn) resp, _ = nsq.ReadResponse(conn) frameType, data, _ = nsq.UnpackResponse(resp) t.Logf("frameType: %d, data: %s", frameType, data) test.Equal(t, frameTypeError, frameType) test.Equal(t, "E_BAD_MESSAGE PUB message too big 105 > 100", string(data)) // need to reconnect conn, err = mustUnixSocketConnectNSQD(addr) test.Nil(t, err) defer conn.Close() // PUB thats empty nsq.Publish(topicName, []byte{}).WriteTo(conn) resp, _ = nsq.ReadResponse(conn) frameType, data, _ = nsq.UnpackResponse(resp) t.Logf("frameType: %d, data: %s", frameType, data) test.Equal(t, frameTypeError, frameType) test.Equal(t, "E_BAD_MESSAGE PUB invalid message body size 0", string(data)) // need to reconnect conn, err = mustUnixSocketConnectNSQD(addr) test.Nil(t, err) defer conn.Close() // MPUB body that's valid mpub := make([][]byte, 5) for i := range mpub { mpub[i] = make([]byte, 100) } cmd, _ := nsq.MultiPublish(topicName, mpub) cmd.WriteTo(conn) resp, _ = nsq.ReadResponse(conn) frameType, data, _ = nsq.UnpackResponse(resp) t.Logf("frameType: %d, data: %s", frameType, data) test.Equal(t, frameTypeResponse, frameType) test.Equal(t, []byte("OK"), data) // MPUB body that's invalid (body too big) mpub = make([][]byte, 11) for i := range mpub { mpub[i] = make([]byte, 100) } cmd, _ = nsq.MultiPublish(topicName, mpub) cmd.WriteTo(conn) resp, _ = nsq.ReadResponse(conn) frameType, data, _ = nsq.UnpackResponse(resp) t.Logf("frameType: %d, data: %s", frameType, data) test.Equal(t, frameTypeError, frameType) test.Equal(t, "E_BAD_BODY MPUB body too big 1148 > 1000", string(data)) // need to reconnect conn, err = mustUnixSocketConnectNSQD(addr) test.Nil(t, err) defer conn.Close() // MPUB that's invalid (one message empty) mpub = make([][]byte, 5) for i := range mpub { mpub[i] = make([]byte, 100) } mpub = append(mpub, []byte{}) cmd, _ = nsq.MultiPublish(topicName, mpub) cmd.WriteTo(conn) resp, _ = nsq.ReadResponse(conn) frameType, data, _ = nsq.UnpackResponse(resp) t.Logf("frameType: %d, data: %s", frameType, data) test.Equal(t, frameTypeError, frameType) test.Equal(t, "E_BAD_MESSAGE MPUB invalid message(5) body size 0", string(data)) // need to reconnect conn, err = mustUnixSocketConnectNSQD(addr) test.Nil(t, err) defer conn.Close() // MPUB body that's invalid (one of the messages is too big) mpub = make([][]byte, 5) for i := range mpub { mpub[i] = make([]byte, 101) } cmd, _ = nsq.MultiPublish(topicName, mpub) cmd.WriteTo(conn) resp, _ = nsq.ReadResponse(conn) frameType, data, _ = nsq.UnpackResponse(resp) t.Logf("frameType: %d, data: %s", frameType, data) test.Equal(t, frameTypeError, frameType) test.Equal(t, "E_BAD_MESSAGE MPUB message too big 101 > 100", string(data)) } func TestUnixSocketDPUB(t *testing.T) { opts := NewOptions() opts.Logger = test.NewTestLogger(t) opts.LogLevel = LOG_DEBUG addr, _, nsqd := mustUnixSocketStartNSQD(opts) defer os.RemoveAll(opts.DataPath) defer nsqd.Exit() conn, err := mustUnixSocketConnectNSQD(addr) test.Nil(t, err) defer conn.Close() topicName := "test_dpub_v2" + strconv.Itoa(int(time.Now().Unix())) identify(t, conn, nil, frameTypeResponse) sub(t, conn, topicName, "ch") // valid nsq.DeferredPublish(topicName, time.Second, make([]byte, 100)).WriteTo(conn) resp, _ := nsq.ReadResponse(conn) frameType, data, _ := nsq.UnpackResponse(resp) t.Logf("frameType: %d, data: %s", frameType, data) test.Equal(t, frameTypeResponse, frameType) test.Equal(t, []byte("OK"), data) time.Sleep(25 * time.Millisecond) ch := nsqd.GetTopic(topicName).GetChannel("ch") ch.deferredMutex.Lock() numDef := len(ch.deferredMessages) ch.deferredMutex.Unlock() test.Equal(t, 1, numDef) test.Equal(t, 1, int(atomic.LoadUint64(&ch.messageCount))) // duration out of range nsq.DeferredPublish(topicName, opts.MaxDeferTimeout+100*time.Millisecond, make([]byte, 100)).WriteTo(conn) resp, _ = nsq.ReadResponse(conn) frameType, data, _ = nsq.UnpackResponse(resp) t.Logf("frameType: %d, data: %s", frameType, data) test.Equal(t, frameTypeError, frameType) test.Equal(t, "E_INVALID DPUB defer timeout 3600100 out of range 0-3600000", string(data)) } func TestUnixSocketTouch(t *testing.T) { opts := NewOptions() opts.Logger = test.NewTestLogger(t) opts.LogLevel = LOG_DEBUG opts.MsgTimeout = 150 * time.Millisecond addr, _, nsqd := mustUnixSocketStartNSQD(opts) defer os.RemoveAll(opts.DataPath) defer nsqd.Exit() topicName := "test_touch" + strconv.Itoa(int(time.Now().Unix())) conn, err := mustUnixSocketConnectNSQD(addr) test.Nil(t, err) defer conn.Close() identify(t, conn, nil, frameTypeResponse) sub(t, conn, topicName, "ch") topic := nsqd.GetTopic(topicName) channel := topic.GetChannel("ch") msg := NewMessage(topic.GenerateID(), []byte("test body")) topic.PutMessage(msg) _, err = nsq.Ready(1).WriteTo(conn) test.Nil(t, err) resp, err := nsq.ReadResponse(conn) test.Nil(t, err) frameType, data, _ := nsq.UnpackResponse(resp) msgOut, _ := decodeMessage(data) test.Equal(t, frameTypeMessage, frameType) test.Equal(t, msg.ID, msgOut.ID) time.Sleep(75 * time.Millisecond) _, err = nsq.Touch(nsq.MessageID(msg.ID)).WriteTo(conn) test.Nil(t, err) time.Sleep(75 * time.Millisecond) _, err = nsq.Finish(nsq.MessageID(msg.ID)).WriteTo(conn) test.Nil(t, err) test.Equal(t, uint64(0), channel.timeoutCount) } func TestUnixSocketMaxRdyCount(t *testing.T) { opts := NewOptions() opts.Logger = test.NewTestLogger(t) opts.LogLevel = LOG_DEBUG opts.MaxRdyCount = 50 addr, _, nsqd := mustUnixSocketStartNSQD(opts) defer os.RemoveAll(opts.DataPath) defer nsqd.Exit() topicName := "test_max_rdy_count" + strconv.Itoa(int(time.Now().Unix())) conn, err := mustUnixSocketConnectNSQD(addr) test.Nil(t, err) defer conn.Close() topic := nsqd.GetTopic(topicName) msg := NewMessage(topic.GenerateID(), []byte("test body")) topic.PutMessage(msg) data := identify(t, conn, nil, frameTypeResponse) r := struct { MaxRdyCount int64 `json:"max_rdy_count"` }{} err = json.Unmarshal(data, &r) test.Nil(t, err) test.Equal(t, int64(50), r.MaxRdyCount) sub(t, conn, topicName, "ch") _, err = nsq.Ready(int(opts.MaxRdyCount)).WriteTo(conn) test.Nil(t, err) resp, err := nsq.ReadResponse(conn) test.Nil(t, err) frameType, data, _ := nsq.UnpackResponse(resp) msgOut, _ := decodeMessage(data) test.Equal(t, frameTypeMessage, frameType) test.Equal(t, msg.ID, msgOut.ID) _, err = nsq.Ready(int(opts.MaxRdyCount) + 1).WriteTo(conn) test.Nil(t, err) resp, err = nsq.ReadResponse(conn) test.Nil(t, err) frameType, data, _ = nsq.UnpackResponse(resp) test.Equal(t, int32(1), frameType) test.Equal(t, "E_INVALID RDY count 51 out of range 0-50", string(data)) } func TestUnixSocketFatalError(t *testing.T) { opts := NewOptions() opts.Logger = test.NewTestLogger(t) addr, _, nsqd := mustUnixSocketStartNSQD(opts) defer os.RemoveAll(opts.DataPath) defer nsqd.Exit() conn, err := mustUnixSocketConnectNSQD(addr) test.Nil(t, err) defer conn.Close() _, err = conn.Write([]byte("ASDF\n")) test.Nil(t, err) resp, err := nsq.ReadResponse(conn) test.Nil(t, err) frameType, data, _ := nsq.UnpackResponse(resp) test.Equal(t, int32(1), frameType) test.Equal(t, "E_INVALID invalid command ASDF", string(data)) _, err = nsq.ReadResponse(conn) test.NotNil(t, err) } func TestUnixSocketOutputBuffering(t *testing.T) { opts := NewOptions() opts.Logger = test.NewTestLogger(t) opts.LogLevel = LOG_DEBUG opts.MaxOutputBufferSize = 512 * 1024 opts.MaxOutputBufferTimeout = time.Second addr, _, nsqd := mustUnixSocketStartNSQD(opts) defer os.RemoveAll(opts.DataPath) defer nsqd.Exit() topicName := "test_output_buffering" + strconv.Itoa(int(time.Now().Unix())) conn, err := mustUnixSocketConnectNSQD(addr) test.Nil(t, err) defer conn.Close() outputBufferSize := 256 * 1024 outputBufferTimeout := 500 topic := nsqd.GetTopic(topicName) msg := NewMessage(topic.GenerateID(), make([]byte, outputBufferSize-1024)) topic.PutMessage(msg) start := time.Now() data := identify(t, conn, map[string]interface{}{ "output_buffer_size": outputBufferSize, "output_buffer_timeout": outputBufferTimeout, }, frameTypeResponse) var decoded map[string]interface{} json.Unmarshal(data, &decoded) v, ok := decoded["output_buffer_size"] test.Equal(t, true, ok) test.Equal(t, outputBufferSize, int(v.(float64))) v = decoded["output_buffer_timeout"] test.Equal(t, outputBufferTimeout, int(v.(float64))) sub(t, conn, topicName, "ch") _, err = nsq.Ready(10).WriteTo(conn) test.Nil(t, err) resp, err := nsq.ReadResponse(conn) test.Nil(t, err) end := time.Now() test.Equal(t, true, int(end.Sub(start)/time.Millisecond) >= outputBufferTimeout) frameType, data, _ := nsq.UnpackResponse(resp) msgOut, _ := decodeMessage(data) test.Equal(t, frameTypeMessage, frameType) test.Equal(t, msg.ID, msgOut.ID) } func TestUnixSocketOutputBufferingValidity(t *testing.T) { opts := NewOptions() opts.Logger = test.NewTestLogger(t) opts.LogLevel = LOG_DEBUG opts.MaxOutputBufferSize = 512 * 1024 opts.MaxOutputBufferTimeout = time.Second addr, _, nsqd := mustUnixSocketStartNSQD(opts) defer os.RemoveAll(opts.DataPath) defer nsqd.Exit() conn, err := mustUnixSocketConnectNSQD(addr) test.Nil(t, err) defer conn.Close() identify(t, conn, map[string]interface{}{ "output_buffer_size": 512 * 1024, "output_buffer_timeout": 1000, }, frameTypeResponse) identify(t, conn, map[string]interface{}{ "output_buffer_size": -1, "output_buffer_timeout": -1, }, frameTypeResponse) identify(t, conn, map[string]interface{}{ "output_buffer_size": 0, "output_buffer_timeout": 0, }, frameTypeResponse) data := identify(t, conn, map[string]interface{}{ "output_buffer_size": 512*1024 + 1, "output_buffer_timeout": 0, }, frameTypeError) test.Equal(t, fmt.Sprintf("E_BAD_BODY IDENTIFY output buffer size (%d) is invalid", 512*1024+1), string(data)) conn, err = mustUnixSocketConnectNSQD(addr) test.Nil(t, err) defer conn.Close() data = identify(t, conn, map[string]interface{}{ "output_buffer_size": 0, "output_buffer_timeout": 1001, }, frameTypeError) test.Equal(t, "E_BAD_BODY IDENTIFY output buffer timeout (1001) is invalid", string(data)) } func TestUnixSocketTLS(t *testing.T) { opts := NewOptions() opts.Logger = test.NewTestLogger(t) opts.LogLevel = LOG_DEBUG opts.TLSCert = "./test/certs/server.pem" opts.TLSKey = "./test/certs/server.key" addr, _, nsqd := mustUnixSocketStartNSQD(opts) defer os.RemoveAll(opts.DataPath) defer nsqd.Exit() conn, err := mustUnixSocketConnectNSQD(addr) test.Nil(t, err) defer conn.Close() data := identify(t, conn, map[string]interface{}{ "tls_v1": true, }, frameTypeResponse) r := struct { TLSv1 bool `json:"tls_v1"` }{} err = json.Unmarshal(data, &r) test.Nil(t, err) test.Equal(t, true, r.TLSv1) tlsConfig := &tls.Config{ InsecureSkipVerify: true, } tlsConn := tls.Client(conn, tlsConfig) err = tlsConn.Handshake() test.Nil(t, err) resp, _ := nsq.ReadResponse(tlsConn) frameType, data, _ := nsq.UnpackResponse(resp) t.Logf("frameType: %d, data: %s", frameType, data) test.Equal(t, frameTypeResponse, frameType) test.Equal(t, []byte("OK"), data) } func TestUnixSocketTLSRequired(t *testing.T) { opts := NewOptions() opts.Logger = test.NewTestLogger(t) opts.LogLevel = LOG_DEBUG opts.TLSCert = "./test/certs/server.pem" opts.TLSKey = "./test/certs/server.key" opts.TLSRequired = TLSRequiredExceptHTTP addr, _, nsqd := mustUnixSocketStartNSQD(opts) defer os.RemoveAll(opts.DataPath) defer nsqd.Exit() topicName := "test_tls_required" + strconv.Itoa(int(time.Now().Unix())) conn, err := mustUnixSocketConnectNSQD(addr) test.Nil(t, err) defer conn.Close() subFail(t, conn, topicName, "ch") conn, err = mustUnixSocketConnectNSQD(addr) test.Nil(t, err) defer conn.Close() data := identify(t, conn, map[string]interface{}{ "tls_v1": true, }, frameTypeResponse) r := struct { TLSv1 bool `json:"tls_v1"` }{} err = json.Unmarshal(data, &r) test.Nil(t, err) test.Equal(t, true, r.TLSv1) tlsConfig := &tls.Config{ InsecureSkipVerify: true, } tlsConn := tls.Client(conn, tlsConfig) err = tlsConn.Handshake() test.Nil(t, err) resp, _ := nsq.ReadResponse(tlsConn) frameType, data, _ := nsq.UnpackResponse(resp) t.Logf("frameType: %d, data: %s", frameType, data) test.Equal(t, frameTypeResponse, frameType) test.Equal(t, []byte("OK"), data) } func TestUnixSocketTLSAuthRequire(t *testing.T) { opts := NewOptions() opts.Logger = test.NewTestLogger(t) opts.LogLevel = LOG_DEBUG opts.TLSCert = "./test/certs/server.pem" opts.TLSKey = "./test/certs/server.key" opts.TLSClientAuthPolicy = "require" addr, _, nsqd := mustUnixSocketStartNSQD(opts) defer os.RemoveAll(opts.DataPath) defer nsqd.Exit() // No Certs conn, err := mustUnixSocketConnectNSQD(addr) test.Nil(t, err) defer conn.Close() data := identify(t, conn, map[string]interface{}{ "tls_v1": true, }, frameTypeResponse) r := struct { TLSv1 bool `json:"tls_v1"` }{} err = json.Unmarshal(data, &r) test.Nil(t, err) test.Equal(t, true, r.TLSv1) tlsConfig := &tls.Config{ InsecureSkipVerify: true, } tlsConn := tls.Client(conn, tlsConfig) _, err = nsq.ReadResponse(tlsConn) test.NotNil(t, err) // With Unsigned Cert conn, err = mustUnixSocketConnectNSQD(addr) test.Nil(t, err) defer conn.Close() data = identify(t, conn, map[string]interface{}{ "tls_v1": true, }, frameTypeResponse) r = struct { TLSv1 bool `json:"tls_v1"` }{} err = json.Unmarshal(data, &r) test.Nil(t, err) test.Equal(t, true, r.TLSv1) cert, err := tls.LoadX509KeyPair("./test/certs/cert.pem", "./test/certs/key.pem") test.Nil(t, err) tlsConfig = &tls.Config{ Certificates: []tls.Certificate{cert}, InsecureSkipVerify: true, } tlsConn = tls.Client(conn, tlsConfig) err = tlsConn.Handshake() test.Nil(t, err) resp, _ := nsq.ReadResponse(tlsConn) frameType, data, _ := nsq.UnpackResponse(resp) t.Logf("frameType: %d, data: %s", frameType, data) test.Equal(t, frameTypeResponse, frameType) test.Equal(t, []byte("OK"), data) } func TestUnixSocketTLSAuthRequireVerify(t *testing.T) { opts := NewOptions() opts.Logger = test.NewTestLogger(t) opts.LogLevel = LOG_DEBUG opts.TLSCert = "./test/certs/server.pem" opts.TLSKey = "./test/certs/server.key" opts.TLSRootCAFile = "./test/certs/ca.pem" opts.TLSClientAuthPolicy = "require-verify" addr, _, nsqd := mustUnixSocketStartNSQD(opts) defer os.RemoveAll(opts.DataPath) defer nsqd.Exit() // with no cert conn, err := mustUnixSocketConnectNSQD(addr) test.Nil(t, err) defer conn.Close() data := identify(t, conn, map[string]interface{}{ "tls_v1": true, }, frameTypeResponse) r := struct { TLSv1 bool `json:"tls_v1"` }{} err = json.Unmarshal(data, &r) test.Nil(t, err) test.Equal(t, true, r.TLSv1) tlsConfig := &tls.Config{ InsecureSkipVerify: true, } tlsConn := tls.Client(conn, tlsConfig) _, err = nsq.ReadResponse(tlsConn) test.NotNil(t, err) // with invalid cert conn, err = mustUnixSocketConnectNSQD(addr) test.Nil(t, err) defer conn.Close() data = identify(t, conn, map[string]interface{}{ "tls_v1": true, }, frameTypeResponse) r = struct { TLSv1 bool `json:"tls_v1"` }{} err = json.Unmarshal(data, &r) test.Nil(t, err) test.Equal(t, true, r.TLSv1) cert, err := tls.LoadX509KeyPair("./test/certs/cert.pem", "./test/certs/key.pem") test.Nil(t, err) tlsConfig = &tls.Config{ Certificates: []tls.Certificate{cert}, InsecureSkipVerify: true, } tlsConn = tls.Client(conn, tlsConfig) _, err = nsq.ReadResponse(tlsConn) test.NotNil(t, err) // with valid cert conn, err = mustUnixSocketConnectNSQD(addr) test.Nil(t, err) defer conn.Close() data = identify(t, conn, map[string]interface{}{ "tls_v1": true, }, frameTypeResponse) r = struct { TLSv1 bool `json:"tls_v1"` }{} err = json.Unmarshal(data, &r) test.Nil(t, err) test.Equal(t, true, r.TLSv1) cert, err = tls.LoadX509KeyPair("./test/certs/client.pem", "./test/certs/client.key") test.Nil(t, err) tlsConfig = &tls.Config{ Certificates: []tls.Certificate{cert}, InsecureSkipVerify: true, } tlsConn = tls.Client(conn, tlsConfig) err = tlsConn.Handshake() test.Nil(t, err) resp, _ := nsq.ReadResponse(tlsConn) frameType, data, _ := nsq.UnpackResponse(resp) t.Logf("frameType: %d, data: %s", frameType, data) test.Equal(t, frameTypeResponse, frameType) test.Equal(t, []byte("OK"), data) } func TestUnixSocketDeflate(t *testing.T) { opts := NewOptions() opts.Logger = test.NewTestLogger(t) opts.LogLevel = LOG_DEBUG opts.DeflateEnabled = true addr, _, nsqd := mustUnixSocketStartNSQD(opts) defer os.RemoveAll(opts.DataPath) defer nsqd.Exit() conn, err := mustUnixSocketConnectNSQD(addr) test.Nil(t, err) defer conn.Close() data := identify(t, conn, map[string]interface{}{ "deflate": true, }, frameTypeResponse) r := struct { Deflate bool `json:"deflate"` }{} err = json.Unmarshal(data, &r) test.Nil(t, err) test.Equal(t, true, r.Deflate) compressConn := flate.NewReader(conn) resp, _ := nsq.ReadResponse(compressConn) frameType, data, _ := nsq.UnpackResponse(resp) t.Logf("frameType: %d, data: %s", frameType, data) test.Equal(t, frameTypeResponse, frameType) test.Equal(t, []byte("OK"), data) } func TestUnixSocketSnappy(t *testing.T) { opts := NewOptions() opts.Logger = test.NewTestLogger(t) opts.LogLevel = LOG_DEBUG opts.SnappyEnabled = true addr, _, nsqd := mustUnixSocketStartNSQD(opts) defer os.RemoveAll(opts.DataPath) defer nsqd.Exit() conn, err := mustUnixSocketConnectNSQD(addr) test.Nil(t, err) defer conn.Close() data := identify(t, conn, map[string]interface{}{ "snappy": true, }, frameTypeResponse) r := struct { Snappy bool `json:"snappy"` }{} err = json.Unmarshal(data, &r) test.Nil(t, err) test.Equal(t, true, r.Snappy) compressConn := snappy.NewReader(conn) resp, _ := nsq.ReadResponse(compressConn) frameType, data, _ := nsq.UnpackResponse(resp) t.Logf("frameType: %d, data: %s", frameType, data) test.Equal(t, frameTypeResponse, frameType) test.Equal(t, []byte("OK"), data) msgBody := make([]byte, 128000) //lint:ignore SA1019 NewWriter is deprecated by NewBufferedWriter, but we don't want to buffer w := snappy.NewWriter(conn) rw := readWriter{compressConn, w} topicName := "test_snappy" + strconv.Itoa(int(time.Now().Unix())) sub(t, rw, topicName, "ch") _, err = nsq.Ready(1).WriteTo(rw) test.Nil(t, err) topic := nsqd.GetTopic(topicName) msg := NewMessage(topic.GenerateID(), msgBody) topic.PutMessage(msg) resp, _ = nsq.ReadResponse(compressConn) frameType, data, _ = nsq.UnpackResponse(resp) msgOut, _ := decodeMessage(data) test.Equal(t, frameTypeMessage, frameType) test.Equal(t, msg.ID, msgOut.ID) test.Equal(t, msg.Body, msgOut.Body) } func TestUnixSocketTLSDeflate(t *testing.T) { opts := NewOptions() opts.Logger = test.NewTestLogger(t) opts.LogLevel = LOG_DEBUG opts.DeflateEnabled = true opts.TLSCert = "./test/certs/cert.pem" opts.TLSKey = "./test/certs/key.pem" addr, _, nsqd := mustUnixSocketStartNSQD(opts) defer os.RemoveAll(opts.DataPath) defer nsqd.Exit() conn, err := mustUnixSocketConnectNSQD(addr) test.Nil(t, err) defer conn.Close() data := identify(t, conn, map[string]interface{}{ "tls_v1": true, "deflate": true, }, frameTypeResponse) r := struct { TLSv1 bool `json:"tls_v1"` Deflate bool `json:"deflate"` }{} err = json.Unmarshal(data, &r) test.Nil(t, err) test.Equal(t, true, r.TLSv1) test.Equal(t, true, r.Deflate) tlsConfig := &tls.Config{ InsecureSkipVerify: true, } tlsConn := tls.Client(conn, tlsConfig) err = tlsConn.Handshake() test.Nil(t, err) resp, _ := nsq.ReadResponse(tlsConn) frameType, data, _ := nsq.UnpackResponse(resp) t.Logf("frameType: %d, data: %s", frameType, data) test.Equal(t, frameTypeResponse, frameType) test.Equal(t, []byte("OK"), data) compressConn := flate.NewReader(tlsConn) resp, _ = nsq.ReadResponse(compressConn) frameType, data, _ = nsq.UnpackResponse(resp) t.Logf("frameType: %d, data: %s", frameType, data) test.Equal(t, frameTypeResponse, frameType) test.Equal(t, []byte("OK"), data) } func TestUnixSocketSampling(t *testing.T) { rand.Seed(time.Now().UTC().UnixNano()) num := 10000 sampleRate := 42 slack := 5 opts := NewOptions() opts.Logger = test.NewTestLogger(t) opts.LogLevel = LOG_DEBUG opts.MaxRdyCount = int64(num) addr, _, nsqd := mustUnixSocketStartNSQD(opts) defer os.RemoveAll(opts.DataPath) defer nsqd.Exit() conn, err := mustUnixSocketConnectNSQD(addr) test.Nil(t, err) defer conn.Close() data := identify(t, conn, map[string]interface{}{ "sample_rate": int32(sampleRate), }, frameTypeResponse) r := struct { SampleRate int32 `json:"sample_rate"` }{} err = json.Unmarshal(data, &r) test.Nil(t, err) test.Equal(t, int32(sampleRate), r.SampleRate) topicName := "test_sampling" + strconv.Itoa(int(time.Now().Unix())) topic := nsqd.GetTopic(topicName) for i := 0; i < num; i++ { msg := NewMessage(topic.GenerateID(), []byte("test body")) topic.PutMessage(msg) } channel := topic.GetChannel("ch") // let the topic drain into the channel time.Sleep(50 * time.Millisecond) sub(t, conn, topicName, "ch") _, err = nsq.Ready(num).WriteTo(conn) test.Nil(t, err) go func() { for { _, err := nsq.ReadResponse(conn) if err != nil { return } } }() doneChan := make(chan int) go func() { for { if channel.Depth() == 0 { close(doneChan) return } time.Sleep(5 * time.Millisecond) } }() <-doneChan channel.inFlightMutex.Lock() numInFlight := len(channel.inFlightMessages) channel.inFlightMutex.Unlock() test.Equal(t, true, numInFlight <= int(float64(num)*float64(sampleRate+slack)/100.0)) test.Equal(t, true, numInFlight >= int(float64(num)*float64(sampleRate-slack)/100.0)) } func TestUnixSocketTLSSnappy(t *testing.T) { opts := NewOptions() opts.Logger = test.NewTestLogger(t) opts.LogLevel = LOG_DEBUG opts.SnappyEnabled = true opts.TLSCert = "./test/certs/cert.pem" opts.TLSKey = "./test/certs/key.pem" addr, _, nsqd := mustUnixSocketStartNSQD(opts) defer os.RemoveAll(opts.DataPath) defer nsqd.Exit() conn, err := mustUnixSocketConnectNSQD(addr) test.Nil(t, err) defer conn.Close() data := identify(t, conn, map[string]interface{}{ "tls_v1": true, "snappy": true, }, frameTypeResponse) r := struct { TLSv1 bool `json:"tls_v1"` Snappy bool `json:"snappy"` }{} err = json.Unmarshal(data, &r) test.Nil(t, err) test.Equal(t, true, r.TLSv1) test.Equal(t, true, r.Snappy) tlsConfig := &tls.Config{ InsecureSkipVerify: true, } tlsConn := tls.Client(conn, tlsConfig) err = tlsConn.Handshake() test.Nil(t, err) resp, _ := nsq.ReadResponse(tlsConn) frameType, data, _ := nsq.UnpackResponse(resp) t.Logf("frameType: %d, data: %s", frameType, data) test.Equal(t, frameTypeResponse, frameType) test.Equal(t, []byte("OK"), data) compressConn := snappy.NewReader(tlsConn) resp, _ = nsq.ReadResponse(compressConn) frameType, data, _ = nsq.UnpackResponse(resp) t.Logf("frameType: %d, data: %s", frameType, data) test.Equal(t, frameTypeResponse, frameType) test.Equal(t, []byte("OK"), data) } func TestUnixSocketClientMsgTimeout(t *testing.T) { opts := NewOptions() opts.Logger = test.NewTestLogger(t) opts.LogLevel = LOG_DEBUG opts.QueueScanRefreshInterval = 100 * time.Millisecond addr, _, nsqd := mustUnixSocketStartNSQD(opts) defer os.RemoveAll(opts.DataPath) defer nsqd.Exit() topicName := "test_cmsg_timeout" + strconv.Itoa(int(time.Now().Unix())) topic := nsqd.GetTopic(topicName) ch := topic.GetChannel("ch") msg := NewMessage(topic.GenerateID(), make([]byte, 100)) topic.PutMessage(msg) // without this the race detector thinks there's a write // to msg.Attempts that races with the read in the protocol's messagePump... // it does not reflect a realistically possible condition topic.PutMessage(NewMessage(topic.GenerateID(), make([]byte, 100))) conn, err := mustUnixSocketConnectNSQD(addr) test.Nil(t, err) defer conn.Close() identify(t, conn, map[string]interface{}{ "msg_timeout": 1000, }, frameTypeResponse) sub(t, conn, topicName, "ch") test.Equal(t, 0, int(atomic.LoadUint64(&ch.timeoutCount))) test.Equal(t, 0, int(atomic.LoadUint64(&ch.requeueCount))) _, err = nsq.Ready(1).WriteTo(conn) test.Nil(t, err) resp, _ := nsq.ReadResponse(conn) _, data, _ := nsq.UnpackResponse(resp) msgOut, _ := decodeMessage(data) test.Equal(t, msg.ID, msgOut.ID) test.Equal(t, msg.Body, msgOut.Body) _, err = nsq.Ready(0).WriteTo(conn) test.Nil(t, err) time.Sleep(1150 * time.Millisecond) test.Equal(t, 1, int(atomic.LoadUint64(&ch.timeoutCount))) test.Equal(t, 0, int(atomic.LoadUint64(&ch.requeueCount))) _, err = nsq.Finish(nsq.MessageID(msgOut.ID)).WriteTo(conn) test.Nil(t, err) resp, _ = nsq.ReadResponse(conn) frameType, data, _ := nsq.UnpackResponse(resp) test.Equal(t, frameTypeError, frameType) test.Equal(t, fmt.Sprintf("E_FIN_FAILED FIN %s failed ID not in flight", msgOut.ID), string(data)) } func TestUnixSocketBadFin(t *testing.T) { opts := NewOptions() opts.Logger = test.NewTestLogger(t) opts.LogLevel = LOG_DEBUG addr, _, nsqd := mustUnixSocketStartNSQD(opts) defer os.RemoveAll(opts.DataPath) defer nsqd.Exit() conn, err := mustUnixSocketConnectNSQD(addr) test.Nil(t, err) defer conn.Close() identify(t, conn, map[string]interface{}{}, frameTypeResponse) sub(t, conn, "test_fin", "ch") fin := nsq.Finish(nsq.MessageID{}) fin.Params[0] = []byte("") _, err = fin.WriteTo(conn) test.Nil(t, err) resp, _ := nsq.ReadResponse(conn) frameType, data, _ := nsq.UnpackResponse(resp) test.Equal(t, frameTypeError, frameType) test.Equal(t, "E_INVALID invalid message ID", string(data)) } func TestUnixSocketReqTimeoutRange(t *testing.T) { opts := NewOptions() opts.Logger = test.NewTestLogger(t) opts.LogLevel = LOG_DEBUG opts.MaxReqTimeout = 1 * time.Minute addr, _, nsqd := mustUnixSocketStartNSQD(opts) defer os.RemoveAll(opts.DataPath) defer nsqd.Exit() topicName := "test_req" + strconv.Itoa(int(time.Now().Unix())) conn, err := mustUnixSocketConnectNSQD(addr) test.Nil(t, err) defer conn.Close() identify(t, conn, nil, frameTypeResponse) sub(t, conn, topicName, "ch") topic := nsqd.GetTopic(topicName) channel := topic.GetChannel("ch") msg := NewMessage(topic.GenerateID(), []byte("test body")) topic.PutMessage(msg) _, err = nsq.Ready(1).WriteTo(conn) test.Nil(t, err) resp, err := nsq.ReadResponse(conn) test.Nil(t, err) frameType, data, _ := nsq.UnpackResponse(resp) msgOut, _ := decodeMessage(data) test.Equal(t, frameTypeMessage, frameType) test.Equal(t, msg.ID, msgOut.ID) _, err = nsq.Requeue(nsq.MessageID(msg.ID), -1).WriteTo(conn) test.Nil(t, err) // It should be immediately available for another attempt resp, err = nsq.ReadResponse(conn) test.Nil(t, err) frameType, data, _ = nsq.UnpackResponse(resp) msgOut, _ = decodeMessage(data) test.Equal(t, frameTypeMessage, frameType) test.Equal(t, msg.ID, msgOut.ID) // The priority (processing time) should be >= this minTs := time.Now().Add(opts.MaxReqTimeout).UnixNano() _, err = nsq.Requeue(nsq.MessageID(msg.ID), opts.MaxReqTimeout*2).WriteTo(conn) test.Nil(t, err) time.Sleep(100 * time.Millisecond) channel.deferredMutex.Lock() pqItem := channel.deferredMessages[msg.ID] channel.deferredMutex.Unlock() test.NotNil(t, pqItem) test.Equal(t, true, pqItem.Priority >= minTs) } func TestUnixSocketIOLoopReturnsClientErrWhenSendFails(t *testing.T) { fakeConn := test.NewFakeNetConn() fakeConn.WriteFunc = func(b []byte) (int, error) { return 0, errors.New("write error") } testUnixSocketIOLoopReturnsClientErr(t, fakeConn) } func TestUnixSocketIOLoopReturnsClientErrWhenSendSucceeds(t *testing.T) { fakeConn := test.NewFakeNetConn() fakeConn.WriteFunc = func(b []byte) (int, error) { return len(b), nil } testUnixSocketIOLoopReturnsClientErr(t, fakeConn) } func testUnixSocketIOLoopReturnsClientErr(t *testing.T, fakeConn test.FakeNetConn) { fakeConn.ReadFunc = func(b []byte) (int, error) { return copy(b, []byte("INVALID_COMMAND\n")), nil } opts := NewOptions() opts.Logger = test.NewTestLogger(t) opts.LogLevel = LOG_DEBUG nsqd, err := New(opts) test.Nil(t, err) prot := &protocolV2{nsqd: nsqd} defer prot.nsqd.Exit() client := prot.NewClient(fakeConn) err = prot.IOLoop(client) test.NotNil(t, err) test.Equal(t, "E_INVALID invalid command INVALID_COMMAND", err.Error()) test.NotNil(t, err.(*protocol.FatalClientErr)) } func BenchmarkUnixSocketProtocolV2Exec(b *testing.B) { b.StopTimer() opts := NewOptions() opts.Logger = test.NewTestLogger(b) nsqd, _ := New(opts) p := &protocolV2{nsqd} c := newClientV2(0, nil, nsqd) params := [][]byte{[]byte("NOP")} b.StartTimer() for i := 0; i < b.N; i++ { p.Exec(c, params) } } func benchmarkUnixSocketProtocolV2PubMultiTopic(b *testing.B, numTopics int) { var wg sync.WaitGroup b.StopTimer() opts := NewOptions() size := 200 batchSize := int(opts.MaxBodySize) / (size + 4) opts.Logger = test.NewTestLogger(b) opts.MemQueueSize = int64(b.N) addr, _, nsqd := mustUnixSocketStartNSQD(opts) defer os.RemoveAll(opts.DataPath) msg := make([]byte, size) batch := make([][]byte, batchSize) for i := range batch { batch[i] = msg } b.SetBytes(int64(len(msg))) b.StartTimer() for j := 0; j < numTopics; j++ { topicName := fmt.Sprintf("bench_v2_pub_multi_topic_%d_%d", j, time.Now().Unix()) wg.Add(1) go func() { conn, err := mustUnixSocketConnectNSQD(addr) if err != nil { panic(err.Error()) } rw := bufio.NewReadWriter(bufio.NewReader(conn), bufio.NewWriter(conn)) num := b.N / numTopics / batchSize wg.Add(1) go func() { for i := 0; i < num; i++ { cmd, _ := nsq.MultiPublish(topicName, batch) _, err := cmd.WriteTo(rw) if err != nil { panic(err.Error()) } err = rw.Flush() if err != nil { panic(err.Error()) } } wg.Done() }() wg.Add(1) go func() { for i := 0; i < num; i++ { resp, err := nsq.ReadResponse(rw) if err != nil { panic(err.Error()) } _, data, _ := nsq.UnpackResponse(resp) if !bytes.Equal(data, []byte("OK")) { panic("invalid response") } } wg.Done() }() wg.Done() }() } wg.Wait() b.StopTimer() nsqd.Exit() } func BenchmarkUnixSocketProtocolV2PubMultiTopic1(b *testing.B) { benchmarkUnixSocketProtocolV2PubMultiTopic(b, 1) } func BenchmarkUnixSocketkProtocolV2PubMultiTopic2(b *testing.B) { benchmarkUnixSocketProtocolV2PubMultiTopic(b, 2) } func BenchmarkUnixSocketProtocolV2PubMultiTopic4(b *testing.B) { benchmarkUnixSocketProtocolV2PubMultiTopic(b, 4) } func BenchmarkUnixSocketProtocolV2PubMultiTopic8(b *testing.B) { benchmarkUnixSocketProtocolV2PubMultiTopic(b, 8) } func BenchmarkUnixSocketProtocolV2PubMultiTopic16(b *testing.B) { benchmarkUnixSocketProtocolV2PubMultiTopic(b, 16) } func BenchmarkUnixSocketProtocolV2PubMultiTopic32(b *testing.B) { benchmarkUnixSocketProtocolV2PubMultiTopic(b, 32) } func benchmarkUnixSocketProtocolV2Pub(b *testing.B, size int) { var wg sync.WaitGroup b.StopTimer() opts := NewOptions() batchSize := int(opts.MaxBodySize) / (size + 4) opts.Logger = test.NewTestLogger(b) opts.MemQueueSize = int64(b.N) addr, _, nsqd := mustUnixSocketStartNSQD(opts) defer os.RemoveAll(opts.DataPath) msg := make([]byte, size) batch := make([][]byte, batchSize) for i := range batch { batch[i] = msg } topicName := "bench_v2_pub" + strconv.Itoa(int(time.Now().Unix())) b.SetBytes(int64(len(msg))) b.StartTimer() for j := 0; j < runtime.GOMAXPROCS(0); j++ { wg.Add(1) go func() { conn, err := mustUnixSocketConnectNSQD(addr) if err != nil { panic(err.Error()) } rw := bufio.NewReadWriter(bufio.NewReader(conn), bufio.NewWriter(conn)) num := b.N / runtime.GOMAXPROCS(0) / batchSize wg.Add(1) go func() { for i := 0; i < num; i++ { cmd, _ := nsq.MultiPublish(topicName, batch) _, err := cmd.WriteTo(rw) if err != nil { panic(err.Error()) } err = rw.Flush() if err != nil { panic(err.Error()) } } wg.Done() }() wg.Add(1) go func() { for i := 0; i < num; i++ { resp, err := nsq.ReadResponse(rw) if err != nil { panic(err.Error()) } _, data, _ := nsq.UnpackResponse(resp) if !bytes.Equal(data, []byte("OK")) { panic("invalid response") } } wg.Done() }() wg.Done() }() } wg.Wait() b.StopTimer() nsqd.Exit() } func BenchmarkUnixSocketProtocolV2Pub256(b *testing.B) { benchmarkUnixSocketProtocolV2Pub(b, 256) } func BenchmarkUnixSocketProtocolV2Pub512(b *testing.B) { benchmarkUnixSocketProtocolV2Pub(b, 512) } func BenchmarkUnixSocketProtocolV2Pub1k(b *testing.B) { benchmarkUnixSocketProtocolV2Pub(b, 1024) } func BenchmarkUnixSocketProtocolV2Pub2k(b *testing.B) { benchmarkUnixSocketProtocolV2Pub(b, 2*1024) } func BenchmarkUnixSocketProtocolV2Pub4k(b *testing.B) { benchmarkUnixSocketProtocolV2Pub(b, 4*1024) } func BenchmarkUnixSocketProtocolV2Pub8k(b *testing.B) { benchmarkUnixSocketProtocolV2Pub(b, 8*1024) } func BenchmarkUnixSocketProtocolV2Pub16k(b *testing.B) { benchmarkUnixSocketProtocolV2Pub(b, 16*1024) } func BenchmarkUnixSocketProtocolV2Pub32k(b *testing.B) { benchmarkUnixSocketProtocolV2Pub(b, 32*1024) } func BenchmarkUnixSocketProtocolV2Pub64k(b *testing.B) { benchmarkUnixSocketProtocolV2Pub(b, 64*1024) } func BenchmarkUnixSocketProtocolV2Pub128k(b *testing.B) { benchmarkUnixSocketProtocolV2Pub(b, 128*1024) } func BenchmarkUnixSocketProtocolV2Pub256k(b *testing.B) { benchmarkUnixSocketProtocolV2Pub(b, 256*1024) } func BenchmarkUnixSocketProtocolV2Pub512k(b *testing.B) { benchmarkUnixSocketProtocolV2Pub(b, 512*1024) } func BenchmarkUnixSocketProtocolV2Pub1m(b *testing.B) { benchmarkUnixSocketProtocolV2Pub(b, 1024*1024) } func benchmarkUnixSocketProtocolV2Sub(b *testing.B, size int) { var wg sync.WaitGroup b.StopTimer() opts := NewOptions() opts.Logger = test.NewTestLogger(b) opts.MemQueueSize = int64(b.N) addr, _, nsqd := mustUnixSocketStartNSQD(opts) defer os.RemoveAll(opts.DataPath) msg := make([]byte, size) topicName := "bench_v2_sub" + strconv.Itoa(b.N) + strconv.Itoa(int(time.Now().Unix())) topic := nsqd.GetTopic(topicName) for i := 0; i < b.N; i++ { msg := NewMessage(topic.GenerateID(), msg) topic.PutMessage(msg) } topic.GetChannel("ch") b.SetBytes(int64(len(msg))) goChan := make(chan int) rdyChan := make(chan int) workers := runtime.GOMAXPROCS(0) for j := 0; j < workers; j++ { wg.Add(1) go func() { subWorker(b.N, workers, addr, topicName, rdyChan, goChan) wg.Done() }() <-rdyChan } b.StartTimer() close(goChan) wg.Wait() b.StopTimer() nsqd.Exit() } func subUnixSocketWorker(n int, workers int, addr net.Addr, topicName string, rdyChan chan int, goChan chan int) { conn, err := mustUnixSocketConnectNSQD(addr) if err != nil { panic(err.Error()) } rw := bufio.NewReadWriter(bufio.NewReader(conn), bufio.NewWriterSize(conn, 65536)) identify(nil, conn, nil, frameTypeResponse) sub(nil, conn, topicName, "ch") rdyCount := int(math.Min(math.Max(float64(n/workers), 1), 2500)) rdyChan <- 1 <-goChan nsq.Ready(rdyCount).WriteTo(rw) rw.Flush() num := n / workers for i := 0; i < num; i++ { resp, err := nsq.ReadResponse(rw) if err != nil { panic(err.Error()) } frameType, data, err := nsq.UnpackResponse(resp) if err != nil { panic(err.Error()) } if frameType != frameTypeMessage { panic("got something else") } msg, err := decodeMessage(data) if err != nil { panic(err.Error()) } nsq.Finish(nsq.MessageID(msg.ID)).WriteTo(rw) if (i+1)%rdyCount == 0 || i+1 == num { if i+1 == num { nsq.Ready(0).WriteTo(conn) } rw.Flush() } } } func BenchmarkUnixSocketProtocolV2Sub256(b *testing.B) { benchmarkUnixSocketProtocolV2Sub(b, 256) } func BenchmarkUnixSocketProtocolV2Sub512(b *testing.B) { benchmarkUnixSocketProtocolV2Sub(b, 512) } func BenchmarkUnixSocketProtocolV2Sub1k(b *testing.B) { benchmarkUnixSocketProtocolV2Sub(b, 1024) } func BenchmarkUnixSocketProtocolV2Sub2k(b *testing.B) { benchmarkUnixSocketProtocolV2Sub(b, 2*1024) } func BenchmarkUnixSocketProtocolV2Sub4k(b *testing.B) { benchmarkUnixSocketProtocolV2Sub(b, 4*1024) } func BenchmarkUnixSocketProtocolV2Sub8k(b *testing.B) { benchmarkUnixSocketProtocolV2Sub(b, 8*1024) } func BenchmarkUnixSocketProtocolV2Sub16k(b *testing.B) { benchmarkUnixSocketProtocolV2Sub(b, 16*1024) } func BenchmarkUnixSocketProtocolV2Sub32k(b *testing.B) { benchmarkUnixSocketProtocolV2Sub(b, 32*1024) } func BenchmarkUnixSocketProtocolV2Sub64k(b *testing.B) { benchmarkUnixSocketProtocolV2Sub(b, 64*1024) } func BenchmarkUnixSocketProtocolV2Sub128k(b *testing.B) { benchmarkUnixSocketProtocolV2Sub(b, 128*1024) } func BenchmarkUnixSocketProtocolV2Sub256k(b *testing.B) { benchmarkUnixSocketProtocolV2Sub(b, 256*1024) } func BenchmarkUnixSocketProtocolV2Sub512k(b *testing.B) { benchmarkUnixSocketProtocolV2Sub(b, 512*1024) } func BenchmarkUnixSocketProtocolV2Sub1m(b *testing.B) { benchmarkUnixSocketProtocolV2Sub(b, 1024*1024) } func benchmarkUnixSocketProtocolV2MultiSub(b *testing.B, num int) { var wg sync.WaitGroup b.StopTimer() opts := NewOptions() opts.Logger = test.NewTestLogger(b) opts.MemQueueSize = int64(b.N) addr, _, nsqd := mustUnixSocketStartNSQD(opts) defer os.RemoveAll(opts.DataPath) msg := make([]byte, 256) b.SetBytes(int64(len(msg) * num)) goChan := make(chan int) rdyChan := make(chan int) workers := runtime.GOMAXPROCS(0) for i := 0; i < num; i++ { topicName := "bench_v2" + strconv.Itoa(b.N) + "_" + strconv.Itoa(i) + "_" + strconv.Itoa(int(time.Now().Unix())) topic := nsqd.GetTopic(topicName) for i := 0; i < b.N; i++ { msg := NewMessage(topic.GenerateID(), msg) topic.PutMessage(msg) } topic.GetChannel("ch") for j := 0; j < workers; j++ { wg.Add(1) go func() { subUnixSocketWorker(b.N, workers, addr, topicName, rdyChan, goChan) wg.Done() }() <-rdyChan } } b.StartTimer() close(goChan) wg.Wait() b.StopTimer() nsqd.Exit() } func BenchmarkUnixSocketProtocolV2MultiSub2(b *testing.B) { benchmarkUnixSocketProtocolV2MultiSub(b, 2) } func BenchmarkUnixSocketProtocolV2MultiSub1(b *testing.B) { benchmarkUnixSocketProtocolV2MultiSub(b, 1) } func BenchmarkUnixSocketProtocolV2MultiSub4(b *testing.B) { benchmarkUnixSocketProtocolV2MultiSub(b, 4) } func BenchmarkUnixSocketProtocolV2MultiSub8(b *testing.B) { benchmarkUnixSocketProtocolV2MultiSub(b, 8) } func BenchmarkUnixSocketProtocolV2MultiSub16(b *testing.B) { benchmarkUnixSocketProtocolV2MultiSub(b, 16) } ================================================ FILE: nsqd/stats.go ================================================ package nsqd import ( "runtime" "sort" "sync/atomic" "github.com/nsqio/nsq/internal/quantile" ) type Stats struct { Topics []TopicStats Producers []ClientStats } type ClientStats interface { String() string } type TopicStats struct { TopicName string `json:"topic_name"` Channels []ChannelStats `json:"channels"` Depth int64 `json:"depth"` BackendDepth int64 `json:"backend_depth"` MessageCount uint64 `json:"message_count"` MessageBytes uint64 `json:"message_bytes"` Paused bool `json:"paused"` E2eProcessingLatency *quantile.Result `json:"e2e_processing_latency"` } func NewTopicStats(t *Topic, channels []ChannelStats) TopicStats { return TopicStats{ TopicName: t.name, Channels: channels, Depth: t.Depth(), BackendDepth: t.backend.Depth(), MessageCount: atomic.LoadUint64(&t.messageCount), MessageBytes: atomic.LoadUint64(&t.messageBytes), Paused: t.IsPaused(), E2eProcessingLatency: t.AggregateChannelE2eProcessingLatency().Result(), } } type ChannelStats struct { ChannelName string `json:"channel_name"` Depth int64 `json:"depth"` BackendDepth int64 `json:"backend_depth"` InFlightCount int `json:"in_flight_count"` DeferredCount int `json:"deferred_count"` MessageCount uint64 `json:"message_count"` ZoneLocalMsgCount uint64 `json:"zone_local_msg_count,omitempty"` RegionLocalMsgCount uint64 `json:"region_local_msg_count,omitempty"` GlobalMsgCount uint64 `json:"global_msg_count,omitempty"` RequeueCount uint64 `json:"requeue_count"` TimeoutCount uint64 `json:"timeout_count"` ClientCount int `json:"client_count"` Clients []ClientStats `json:"clients"` Paused bool `json:"paused"` E2eProcessingLatency *quantile.Result `json:"e2e_processing_latency"` } func NewChannelStats(c *Channel, clients []ClientStats, clientCount int) ChannelStats { c.inFlightMutex.Lock() inflight := len(c.inFlightMessages) c.inFlightMutex.Unlock() c.deferredMutex.Lock() deferred := len(c.deferredMessages) c.deferredMutex.Unlock() return ChannelStats{ ChannelName: c.name, Depth: c.Depth(), BackendDepth: c.backend.Depth(), InFlightCount: inflight, DeferredCount: deferred, MessageCount: atomic.LoadUint64(&c.messageCount), ZoneLocalMsgCount: atomic.LoadUint64(&c.zoneLocalMsgCount), RegionLocalMsgCount: atomic.LoadUint64(&c.regionLocalMsgCount), GlobalMsgCount: atomic.LoadUint64(&c.globalMsgCount), RequeueCount: atomic.LoadUint64(&c.requeueCount), TimeoutCount: atomic.LoadUint64(&c.timeoutCount), ClientCount: clientCount, Clients: clients, Paused: c.IsPaused(), E2eProcessingLatency: c.e2eProcessingLatencyStream.Result(), } } type Topics []*Topic func (t Topics) Len() int { return len(t) } func (t Topics) Swap(i, j int) { t[i], t[j] = t[j], t[i] } type TopicsByName struct { Topics } func (t TopicsByName) Less(i, j int) bool { return t.Topics[i].name < t.Topics[j].name } type Channels []*Channel func (c Channels) Len() int { return len(c) } func (c Channels) Swap(i, j int) { c[i], c[j] = c[j], c[i] } type ChannelsByName struct { Channels } func (c ChannelsByName) Less(i, j int) bool { return c.Channels[i].name < c.Channels[j].name } func (n *NSQD) GetStats(topic string, channel string, includeClients bool) Stats { var stats Stats n.RLock() var realTopics []*Topic if topic == "" { realTopics = make([]*Topic, 0, len(n.topicMap)) for _, t := range n.topicMap { realTopics = append(realTopics, t) } } else if val, exists := n.topicMap[topic]; exists { realTopics = []*Topic{val} } else { n.RUnlock() return stats } n.RUnlock() sort.Sort(TopicsByName{realTopics}) topics := make([]TopicStats, 0, len(realTopics)) for _, t := range realTopics { t.RLock() var realChannels []*Channel if channel == "" { realChannels = make([]*Channel, 0, len(t.channelMap)) for _, c := range t.channelMap { realChannels = append(realChannels, c) } } else if val, exists := t.channelMap[channel]; exists { realChannels = []*Channel{val} } else { t.RUnlock() continue } t.RUnlock() sort.Sort(ChannelsByName{realChannels}) channels := make([]ChannelStats, 0, len(realChannels)) for _, c := range realChannels { var clients []ClientStats var clientCount int c.RLock() if includeClients { clients = make([]ClientStats, 0, len(c.clients)) for _, client := range c.clients { clients = append(clients, client.Stats(topic)) } } clientCount = len(c.clients) c.RUnlock() channels = append(channels, NewChannelStats(c, clients, clientCount)) } topics = append(topics, NewTopicStats(t, channels)) } stats.Topics = topics if includeClients { var producerStats []ClientStats n.tcpServer.conns.Range(func(k, v interface{}) bool { c := v.(Client) if c.Type() == typeProducer { producerStats = append(producerStats, c.Stats(topic)) } return true }) stats.Producers = producerStats } return stats } type memStats struct { HeapObjects uint64 `json:"heap_objects"` HeapIdleBytes uint64 `json:"heap_idle_bytes"` HeapInUseBytes uint64 `json:"heap_in_use_bytes"` HeapReleasedBytes uint64 `json:"heap_released_bytes"` GCPauseUsec100 uint64 `json:"gc_pause_usec_100"` GCPauseUsec99 uint64 `json:"gc_pause_usec_99"` GCPauseUsec95 uint64 `json:"gc_pause_usec_95"` NextGCBytes uint64 `json:"next_gc_bytes"` GCTotalRuns uint32 `json:"gc_total_runs"` } func getMemStats() memStats { var ms runtime.MemStats runtime.ReadMemStats(&ms) // sort the GC pause array length := len(ms.PauseNs) if int(ms.NumGC) < length { length = int(ms.NumGC) } gcPauses := make(Uint64Slice, length) copy(gcPauses, ms.PauseNs[:length]) sort.Sort(gcPauses) return memStats{ ms.HeapObjects, ms.HeapIdle, ms.HeapInuse, ms.HeapReleased, percentile(100.0, gcPauses, len(gcPauses)) / 1000, percentile(99.0, gcPauses, len(gcPauses)) / 1000, percentile(95.0, gcPauses, len(gcPauses)) / 1000, ms.NextGC, ms.NumGC, } } ================================================ FILE: nsqd/stats_test.go ================================================ package nsqd import ( "encoding/json" "fmt" "os" "strconv" "sync" "testing" "time" "github.com/golang/snappy" "github.com/nsqio/nsq/internal/http_api" "github.com/nsqio/nsq/internal/test" ) func TestStats(t *testing.T) { opts := NewOptions() opts.Logger = test.NewTestLogger(t) tcpAddr, _, nsqd := mustStartNSQD(opts) defer os.RemoveAll(opts.DataPath) defer nsqd.Exit() topicName := "test_stats" + strconv.Itoa(int(time.Now().Unix())) topic := nsqd.GetTopic(topicName) msg := NewMessage(topic.GenerateID(), []byte("test body")) topic.PutMessage(msg) accompanyTopicName := "accompany_test_stats" + strconv.Itoa(int(time.Now().Unix())) accompanyTopic := nsqd.GetTopic(accompanyTopicName) msg = NewMessage(accompanyTopic.GenerateID(), []byte("accompany test body")) accompanyTopic.PutMessage(msg) conn, err := mustConnectNSQD(tcpAddr) test.Nil(t, err) defer conn.Close() identify(t, conn, nil, frameTypeResponse) sub(t, conn, topicName, "ch") stats := nsqd.GetStats(topicName, "ch", true).Topics t.Logf("stats: %+v", stats) test.Equal(t, 1, len(stats)) test.Equal(t, 1, len(stats[0].Channels)) test.Equal(t, 1, len(stats[0].Channels[0].Clients)) test.Equal(t, 1, stats[0].Channels[0].ClientCount) stats = nsqd.GetStats(topicName, "ch", false).Topics t.Logf("stats: %+v", stats) test.Equal(t, 1, len(stats)) test.Equal(t, 1, len(stats[0].Channels)) test.Equal(t, 0, len(stats[0].Channels[0].Clients)) test.Equal(t, 1, stats[0].Channels[0].ClientCount) stats = nsqd.GetStats(topicName, "none_exist_channel", false).Topics t.Logf("stats: %+v", stats) test.Equal(t, 0, len(stats)) stats = nsqd.GetStats("none_exist_topic", "none_exist_channel", false).Topics t.Logf("stats: %+v", stats) test.Equal(t, 0, len(stats)) } func TestClientAttributes(t *testing.T) { userAgent := "Test User Agent" opts := NewOptions() opts.Logger = test.NewTestLogger(t) opts.LogLevel = LOG_DEBUG opts.SnappyEnabled = true tcpAddr, httpAddr, nsqd := mustStartNSQD(opts) defer os.RemoveAll(opts.DataPath) defer nsqd.Exit() conn, err := mustConnectNSQD(tcpAddr) test.Nil(t, err) defer conn.Close() data := identify(t, conn, map[string]interface{}{ "snappy": true, "user_agent": userAgent, }, frameTypeResponse) resp := struct { Snappy bool `json:"snappy"` UserAgent string `json:"user_agent"` }{} err = json.Unmarshal(data, &resp) test.Nil(t, err) test.Equal(t, true, resp.Snappy) r := snappy.NewReader(conn) //lint:ignore SA1019 NewWriter is deprecated by NewBufferedWriter, but we don't want to buffer w := snappy.NewWriter(conn) readValidate(t, r, frameTypeResponse, "OK") topicName := "test_client_attributes" + strconv.Itoa(int(time.Now().Unix())) sub(t, readWriter{r, w}, topicName, "ch") var d struct { Topics []struct { Channels []struct { Clients []struct { UserAgent string `json:"user_agent"` Snappy bool `json:"snappy"` } `json:"clients"` } `json:"channels"` } `json:"topics"` } endpoint := fmt.Sprintf("http://%s/stats?format=json", httpAddr) err = http_api.NewClient(nil, ConnectTimeout, RequestTimeout).GETV1(endpoint, &d) test.Nil(t, err) test.Equal(t, userAgent, d.Topics[0].Channels[0].Clients[0].UserAgent) test.Equal(t, true, d.Topics[0].Channels[0].Clients[0].Snappy) } func TestStatsChannelLocking(t *testing.T) { opts := NewOptions() opts.Logger = test.NewTestLogger(t) _, _, nsqd := mustStartNSQD(opts) defer os.RemoveAll(opts.DataPath) defer nsqd.Exit() topicName := "test_channel_empty" + strconv.Itoa(int(time.Now().Unix())) topic := nsqd.GetTopic(topicName) channel := topic.GetChannel("channel") var wg sync.WaitGroup wg.Add(2) go func() { for i := 0; i < 25; i++ { msg := NewMessage(topic.GenerateID(), []byte("test")) topic.PutMessage(msg) channel.StartInFlightTimeout(msg, 0, opts.MsgTimeout) } wg.Done() }() go func() { for i := 0; i < 25; i++ { nsqd.GetStats("", "", true) } wg.Done() }() wg.Wait() stats := nsqd.GetStats(topicName, "channel", false).Topics t.Logf("stats: %+v", stats) test.Equal(t, 1, len(stats)) test.Equal(t, 1, len(stats[0].Channels)) test.Equal(t, 25, stats[0].Channels[0].InFlightCount) } ================================================ FILE: nsqd/statsd.go ================================================ package nsqd import ( "fmt" "math" "net" "strings" "time" "github.com/nsqio/nsq/internal/statsd" "github.com/nsqio/nsq/internal/writers" ) type Uint64Slice []uint64 func (s Uint64Slice) Len() int { return len(s) } func (s Uint64Slice) Swap(i, j int) { s[i], s[j] = s[j], s[i] } func (s Uint64Slice) Less(i, j int) bool { return s[i] < s[j] } func (n *NSQD) statsdLoop() { var lastMemStats memStats var lastStats Stats interval := n.getOpts().StatsdInterval ticker := time.NewTicker(interval) for { select { case <-n.exitChan: goto exit case <-ticker.C: addr := n.getOpts().StatsdAddress prefix := n.getOpts().StatsdPrefix excludeEphemeral := n.getOpts().StatsdExcludeEphemeral conn, err := net.DialTimeout("udp", addr, time.Second) if err != nil { n.logf(LOG_ERROR, "failed to create UDP socket to statsd(%s)", addr) continue } sw := writers.NewSpreadWriter(conn, interval-time.Second, n.exitChan) bw := writers.NewBoundaryBufferedWriter(sw, n.getOpts().StatsdUDPPacketSize) client := statsd.NewClient(bw, prefix) n.logf(LOG_INFO, "STATSD: pushing stats to %s", addr) stats := n.GetStats("", "", false) for _, topic := range stats.Topics { if excludeEphemeral && strings.HasSuffix(topic.TopicName, "#ephemeral") { continue } // try to find the topic in the last collection lastTopic := TopicStats{} for _, checkTopic := range lastStats.Topics { if topic.TopicName == checkTopic.TopicName { lastTopic = checkTopic break } } diff := topic.MessageCount - lastTopic.MessageCount stat := fmt.Sprintf("topic.%s.message_count", topic.TopicName) client.Incr(stat, int64(diff)) diff = topic.MessageBytes - lastTopic.MessageBytes stat = fmt.Sprintf("topic.%s.message_bytes", topic.TopicName) client.Incr(stat, int64(diff)) stat = fmt.Sprintf("topic.%s.depth", topic.TopicName) client.Gauge(stat, topic.Depth) stat = fmt.Sprintf("topic.%s.backend_depth", topic.TopicName) client.Gauge(stat, topic.BackendDepth) for _, item := range topic.E2eProcessingLatency.Percentiles { stat = fmt.Sprintf("topic.%s.e2e_processing_latency_%.0f", topic.TopicName, item["quantile"]*100.0) // We can cast the value to int64 since a value of 1 is the // minimum resolution we will have, so there is no loss of // accuracy client.Gauge(stat, int64(item["value"])) } for _, channel := range topic.Channels { if excludeEphemeral && strings.HasSuffix(channel.ChannelName, "#ephemeral") { continue } // try to find the channel in the last collection lastChannel := ChannelStats{} for _, checkChannel := range lastTopic.Channels { if channel.ChannelName == checkChannel.ChannelName { lastChannel = checkChannel break } } diff := channel.MessageCount - lastChannel.MessageCount stat := fmt.Sprintf("topic.%s.channel.%s.message_count", topic.TopicName, channel.ChannelName) client.Incr(stat, int64(diff)) stat = fmt.Sprintf("topic.%s.channel.%s.depth", topic.TopicName, channel.ChannelName) client.Gauge(stat, channel.Depth) stat = fmt.Sprintf("topic.%s.channel.%s.backend_depth", topic.TopicName, channel.ChannelName) client.Gauge(stat, channel.BackendDepth) stat = fmt.Sprintf("topic.%s.channel.%s.in_flight_count", topic.TopicName, channel.ChannelName) client.Gauge(stat, int64(channel.InFlightCount)) stat = fmt.Sprintf("topic.%s.channel.%s.deferred_count", topic.TopicName, channel.ChannelName) client.Gauge(stat, int64(channel.DeferredCount)) diff = channel.RequeueCount - lastChannel.RequeueCount stat = fmt.Sprintf("topic.%s.channel.%s.requeue_count", topic.TopicName, channel.ChannelName) client.Incr(stat, int64(diff)) diff = channel.TimeoutCount - lastChannel.TimeoutCount stat = fmt.Sprintf("topic.%s.channel.%s.timeout_count", topic.TopicName, channel.ChannelName) client.Incr(stat, int64(diff)) stat = fmt.Sprintf("topic.%s.channel.%s.clients", topic.TopicName, channel.ChannelName) client.Gauge(stat, int64(channel.ClientCount)) for _, item := range channel.E2eProcessingLatency.Percentiles { stat = fmt.Sprintf("topic.%s.channel.%s.e2e_processing_latency_%.0f", topic.TopicName, channel.ChannelName, item["quantile"]*100.0) client.Gauge(stat, int64(item["value"])) } } } lastStats = stats if n.getOpts().StatsdMemStats { ms := getMemStats() client.Gauge("mem.heap_objects", int64(ms.HeapObjects)) client.Gauge("mem.heap_idle_bytes", int64(ms.HeapIdleBytes)) client.Gauge("mem.heap_in_use_bytes", int64(ms.HeapInUseBytes)) client.Gauge("mem.heap_released_bytes", int64(ms.HeapReleasedBytes)) client.Gauge("mem.gc_pause_usec_100", int64(ms.GCPauseUsec100)) client.Gauge("mem.gc_pause_usec_99", int64(ms.GCPauseUsec99)) client.Gauge("mem.gc_pause_usec_95", int64(ms.GCPauseUsec95)) client.Gauge("mem.next_gc_bytes", int64(ms.NextGCBytes)) client.Incr("mem.gc_runs", int64(ms.GCTotalRuns-lastMemStats.GCTotalRuns)) lastMemStats = ms } bw.Flush() sw.Flush() conn.Close() } } exit: ticker.Stop() n.logf(LOG_INFO, "STATSD: closing") } func percentile(perc float64, arr []uint64, length int) uint64 { if length == 0 { return 0 } indexOfPerc := int(math.Floor(((perc / 100.0) * float64(length)) + 0.5)) if indexOfPerc >= length { indexOfPerc = length - 1 } return arr[indexOfPerc] } ================================================ FILE: nsqd/tcp.go ================================================ package nsqd import ( "io" "net" "sync" "github.com/nsqio/nsq/internal/protocol" ) const ( typeConsumer = iota typeProducer ) type Client interface { Type() int Stats(string) ClientStats } type tcpServer struct { nsqd *NSQD conns sync.Map } func (p *tcpServer) Handle(conn net.Conn) { p.nsqd.logf(LOG_INFO, "TCP: new client(%s)", conn.RemoteAddr()) // The client should initialize itself by sending a 4 byte sequence indicating // the version of the protocol that it intends to communicate, this will allow us // to gracefully upgrade the protocol away from text/line oriented to whatever... buf := make([]byte, 4) _, err := io.ReadFull(conn, buf) if err != nil { p.nsqd.logf(LOG_ERROR, "failed to read protocol version - %s", err) conn.Close() return } protocolMagic := string(buf) p.nsqd.logf(LOG_INFO, "CLIENT(%s): desired protocol magic '%s'", conn.RemoteAddr(), protocolMagic) var prot protocol.Protocol switch protocolMagic { case " V2": prot = &protocolV2{nsqd: p.nsqd} default: protocol.SendFramedResponse(conn, frameTypeError, []byte("E_BAD_PROTOCOL")) conn.Close() p.nsqd.logf(LOG_ERROR, "client(%s) bad protocol magic '%s'", conn.RemoteAddr(), protocolMagic) return } client := prot.NewClient(conn) p.conns.Store(conn.RemoteAddr(), client) err = prot.IOLoop(client) if err != nil { p.nsqd.logf(LOG_ERROR, "client(%s) - %s", conn.RemoteAddr(), err) } p.conns.Delete(conn.RemoteAddr()) client.Close() } func (p *tcpServer) Close() { p.conns.Range(func(k, v interface{}) bool { v.(protocol.Client).Close() return true }) } ================================================ FILE: nsqd/test/cert.sh ================================================ #!/bin/bash # ./cert.sh foo@foo.com 127.0.0.1 # Found: https://gist.github.com/ncw/9253562#file-makecert-sh if [ "$1" == "" ]; then echo "Need email as argument" exit 1 fi if [ "$2" == "" ]; then echo "Need CN as argument" exit 1 fi PRIVKEY="test" EMAIL=$1 CN=$2 rm -rf tmp mkdir tmp cd tmp echo "make CA" openssl req -new -x509 -days 3650 -keyout ca.key -out ca.pem \ -config ../openssl.conf -extensions ca \ -subj "/CN=ca" \ -passout pass:$PRIVKEY echo "make server cert" openssl genrsa -out server.key 2048 openssl req -new -sha256 -key server.key -out server.req \ -subj "/emailAddress=${EMAIL}/C=DE/ST=NRW/L=Earth/O=Random Company/OU=IT/CN=${CN}" openssl x509 -req -days 3650 -sha256 -in server.req -CA ca.pem -CAkey ca.key -CAcreateserial -passin pass:$PRIVKEY -out server.pem \ -extfile ../openssl.conf -extensions server echo "make client cert" openssl genrsa -out client.key 2048 openssl req -new -sha256 -key client.key -out client.req \ -subj "/emailAddress=${EMAIL}/C=DE/ST=NRW/L=Earth/O=Random Company/OU=IT/CN=${CN}" openssl x509 -req -days 3650 -sha256 -in client.req -CA ca.pem -CAkey ca.key -CAserial ca.srl -passin pass:$PRIVKEY -out client.pem \ -extfile ../openssl.conf -extensions client cd .. mv tmp/* certs rm -rf tmp ================================================ FILE: nsqd/test/certs/ca.key ================================================ -----BEGIN RSA PRIVATE KEY----- Proc-Type: 4,ENCRYPTED DEK-Info: DES-EDE3-CBC,D6208487450BDF1C tucU/4j6agGQlW60D8V3Zr/QHcLhhyFagres1gGdWfGIqluNZb7omki/XidHXJSG eB9vV2Xb12/8umc31e7Mnmn9hd34230v/KlAnJ4ukDpJpbmjnEx3F9uiqYFi/yxQ avSsfF6Tsh3XOh3Oe27I/xfYx37g6Agd+EQEJ1hvWvygMIJMTDMP5ZaFoZANtFLy hDEZ6woJSn9avF/L+1GW8jl2aI1QbdKkK0jDHgFAwUI4sjWeXvEQNNYY3trTIoMo wab3vi+4XziFONbS4OZrZUYfZPB5YOFbtT2whzggp2HdSTiu48/Ld3N8SjuMrKfm uR+nd+ovQ5kVWHInzWAIXSyPhgR9ZY8eyXaHNJJfzNu3HY72lfzD/NtZfacMRBr6 M3Wg/OKPS7ZrtqCWkY9P3KK9Cul8Jzy229fSqHo8Rg4= -----END RSA PRIVATE KEY----- ================================================ FILE: nsqd/test/certs/ca.pem ================================================ -----BEGIN CERTIFICATE----- MIIBVDCB/6ADAgECAgkAvHG4Z/7nX/gwDQYJKoZIhvcNAQEFBQAwDTELMAkGA1UE AwwCY2EwHhcNMTcwOTE2MTc0NDE0WhcNMjcwOTE0MTc0NDE0WjANMQswCQYDVQQD DAJjYTBcMA0GCSqGSIb3DQEBAQUAA0sAMEgCQQDJAM3Tr1BoxlLJtTy0oRcp93dT 9hhHwms8P1V3k2FpXYRS4deUo+uwcAM9KGDt9VMXVBEchtI4VYTvLgatBPUBAgMB AAGjQjBAMA8GA1UdEwEB/wQFMAMBAf8wHQYDVR0OBBYEFMyLx7rjKBe/xZQLnVzI uqNNVzxRMA4GA1UdDwEB/wQEAwIBBjANBgkqhkiG9w0BAQUFAANBAJiN1XgPPlQ2 z7PhtzXStaz/BJVqhD7g9fsZsmoPX4ifDsTfzUsRB56Aq/NTsKiIYQkFPHH0donG ++a5ZVWjgYk= -----END CERTIFICATE----- ================================================ FILE: nsqd/test/certs/ca.srl ================================================ 91418D04995922E7 ================================================ FILE: nsqd/test/certs/cert.pem ================================================ -----BEGIN CERTIFICATE----- MIIEbjCCA1agAwIBAgIJAK6x7y6AwBmLMA0GCSqGSIb3DQEBBQUAMIGAMQswCQYD VQQGEwJVUzERMA8GA1UECBMITmV3IFlvcmsxFjAUBgNVBAcTDU5ldyBZb3JrIENp dHkxDDAKBgNVBAoTA05TUTETMBEGA1UEAxMKdGVzdC5sb2NhbDEjMCEGCSqGSIb3 DQEJARYUbXJlaWZlcnNvbkBnbWFpbC5jb20wHhcNMTMwNjI4MDA0MzQ4WhcNMTYw NDE3MDA0MzQ4WjCBgDELMAkGA1UEBhMCVVMxETAPBgNVBAgTCE5ldyBZb3JrMRYw FAYDVQQHEw1OZXcgWW9yayBDaXR5MQwwCgYDVQQKEwNOU1ExEzARBgNVBAMTCnRl c3QubG9jYWwxIzAhBgkqhkiG9w0BCQEWFG1yZWlmZXJzb25AZ21haWwuY29tMIIB IjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAnX0KB+svwy+yHU2qggz/EaGg craKShagKo+9M9y5HLM852ngk5c+t+tJJbx3N954Wr1FXBuGIv1ltU05rU4zhvBS 25tVP1UIEnT5pBt2TeetLkl199Y7fxh1hKmnwJMG3fy3VZdNXEndBombXMmtXpQY shuEJHKeUNDbQKz5X+GjEdkTPO/HY/VMHsxS23pbSimQozMg3hvLIdgv0aS3QECz ydZBgTPThy3uDtHIuCpxCwXd/vDF68ATlYgo3h3lh2vxNwM/pjklIUhzMh4XaKQF 7m3/0KbtUcXfy0QHueeuMr11E9MAFNyRN4xf9Fk1yB97KJ3PJBTC5WD/m1nW+QID AQABo4HoMIHlMB0GA1UdDgQWBBR3HMBws4lmYYSIgwoZsfW+bbgaMjCBtQYDVR0j BIGtMIGqgBR3HMBws4lmYYSIgwoZsfW+bbgaMqGBhqSBgzCBgDELMAkGA1UEBhMC VVMxETAPBgNVBAgTCE5ldyBZb3JrMRYwFAYDVQQHEw1OZXcgWW9yayBDaXR5MQww CgYDVQQKEwNOU1ExEzARBgNVBAMTCnRlc3QubG9jYWwxIzAhBgkqhkiG9w0BCQEW FG1yZWlmZXJzb25AZ21haWwuY29tggkArrHvLoDAGYswDAYDVR0TBAUwAwEB/zAN BgkqhkiG9w0BAQUFAAOCAQEANOYTbanW2iyV1v4oYpcM/y3TWcQKzSME8D2SGFZb dbMYU81hH3TTlQdvyeh3FAcdjhKE8Xi/RfNNjEslTBscdKXePGpZg6eXRNJzPP5K KZPf5u6tcpAeUOKrMqbGwbE+h2QixxG1EoVQtE421szsU2P7nHRTdHzKFRnOerfl Phm3NocR0P40Rv7WKdxpOvqc+XKf0onTruoVYoPWGpwcLixCG0zu4ZQ23/L/Dy18 4u70Hbq6O/6kq9FBFaDNp3IhiEdu2Cq6ZplU6bL9XDF27KIEErHwtuqBHVlMG+zB oH/k9vZvwH7OwAjHdKp+1yeZFLYC8K5hjFIHqcdwpZCNIg== -----END CERTIFICATE----- ================================================ FILE: nsqd/test/certs/client.key ================================================ -----BEGIN RSA PRIVATE KEY----- MIIEowIBAAKCAQEAuMz5IJ01Wqaj2qJAU7o2aoc2iP1BRhTba+cWrsjbZA+6kXAk 19Dlz/nNK8gxsBMl3rzTGWXKUm/rKBDp680zJkj8nzcGApGlzbDoqXym8ixOniOC aH/RODPC8OtD2UozeXlUOQWK+ufkMvmVBc/IeJyFkcm7jhX+LHVFn+3iWLRNbzGm hbf8oNHvybS7HSuV3OYiHDJxA0d6IgozqVOYVSqdDabQ2FN/8nKzpMHjwMpfT008 5hUhr72E5XuJ3biPUwoOrRETSa1/EQcAEH0wSwHU1AGdnGh3/5ynVupfef0qkyqw NseYs9U++IUFXqA1IuEmAlJn5QczAHjPQe8xBQIDAQABAoIBAGx4CyZEgCOUOgrD P2SloPkIIk9n7x82cNA11I+E35kszkI9g7KVL77SDcZL/DYwFwNU68c1gvq+LFXZ D6RTTlmDb5v4TPPHD33a/8UzoD33GbIif5HcrC4D28FTJgDtV6dOOsw5X6kD4WK2 Me02V6HLpW677PVqHUV1FAfaNgf/1SIhJCechYSO+3pKPtli8DnsmOd336wNtWZR Q3Ctm63Eq7AiLmGLOkLYtll9Kr6WfCSFHBmqJg9MsQEfINXMCOB54Z+xkoijjJLO zNSk/3XvppB1JpmAusbo9Mq03Ci9JdinYSKxwVOG6e+5cosrOHpDQyZIWa87ZTpk kjLCgEECgYEA33JCGTEwaqfAuYOspCZvi2Mqz7t6QiwvwxgDf4JFLgDQArJfzKlC bAzaSU8bT+SN96CF9LU4XekN5/paGZAFjWxraPkK0oL28kQrtzfEQv2XL6GFoNAh WsQtoaGddL9VPWlCiHeueYERIDNrZCaVeAx6OV6W3CEwGOkRiQyntlUCgYEA07lf LpxF8aAgMToq3++D5RGAXaT2PYxhZZFb5Y2tS7a4T0ulQwXQi4H2dObGJ32zCjFG ls1H91mNKGBbyKnF1jT/dt/acqymvipASm1xq6uysIllA4xp56sDnJgG9bnXjYFY m2yVNIoIQGZ7jp7e6t1KQ9San6V8asWlvBLXX/ECgYEAuhF9dVj+xnH3DQTXSMIw 9NOZnO6zelMtWrqufwnN7ecDUJuVJupzw2JYi99yEO90QRbNNd+KlrkxuVFCojLK TOBR+VIZbv9cAJZACQxJRLfDpAhPLIDkpZ7jmMrqQYPqyX7TxqxTAB84UaY/8WAn 65YIWamo2ppQYQ4Eaim9pxkCgYAMwfW/TEFWruxhqvycY8VRzz0p51/DE6tmwFyG N4RCtK7kcE1z/Wy0i087ehBknslkCtYTDimQ+P9teGjvbXNzVdwy4Ig8MrUVblxT X8birkTlKFJC5XoYMJDWJb79nYYki6+4JdHTyaF3p/U4AdCy3ES2U6BBkGov0NsM uyHpMQKBgGc9wIGHwAGLg3iVHGXbXELdpzVztLRC7D12TPfdtGBLTD5NH5jAx8K/ w61+l4tzFcK3jfWzwdme277cBgfq/aw2DDt07vkOvPzhUwVZiE39bfddfIvyed99 XzEMf8THh5wdm6mLsOVgdfOfFcDi5ReIv1/7+eOd/MQi8TxIVfAu -----END RSA PRIVATE KEY----- ================================================ FILE: nsqd/test/certs/client.pem ================================================ -----BEGIN CERTIFICATE----- MIICyzCCAnWgAwIBAgIJAJFBjQSZWSLnMA0GCSqGSIb3DQEBCwUAMA0xCzAJBgNV BAMMAmNhMB4XDTE3MDkxNjE3NDQxNFoXDTI3MDkxNDE3NDQxNFowgYcxIzAhBgkq hkiG9w0BCQEWFG1yZWlmZXJzb25AZ21haWwuY29tMQswCQYDVQQGEwJERTEMMAoG A1UECBMDTlJXMQ4wDAYDVQQHEwVFYXJ0aDEXMBUGA1UEChMOUmFuZG9tIENvbXBh bnkxCzAJBgNVBAsTAklUMQ8wDQYDVQQDEwZuc3EuaW8wggEiMA0GCSqGSIb3DQEB AQUAA4IBDwAwggEKAoIBAQC4zPkgnTVapqPaokBTujZqhzaI/UFGFNtr5xauyNtk D7qRcCTX0OXP+c0ryDGwEyXevNMZZcpSb+soEOnrzTMmSPyfNwYCkaXNsOipfKby LE6eI4Jof9E4M8Lw60PZSjN5eVQ5BYr65+Qy+ZUFz8h4nIWRybuOFf4sdUWf7eJY tE1vMaaFt/yg0e/JtLsdK5Xc5iIcMnEDR3oiCjOpU5hVKp0NptDYU3/ycrOkwePA yl9PTTzmFSGvvYTle4nduI9TCg6tERNJrX8RBwAQfTBLAdTUAZ2caHf/nKdW6l95 /SqTKrA2x5iz1T74hQVeoDUi4SYCUmflBzMAeM9B7zEFAgMBAAGjdTBzMAwGA1Ud EwEB/wQCMAAwHQYDVR0OBBYEFMzYXiTD7moi0oXqSQ5D2LsO51GXMB8GA1UdIwQY MBaAFMyLx7rjKBe/xZQLnVzIuqNNVzxRMA4GA1UdDwEB/wQEAwIFoDATBgNVHSUE DDAKBggrBgEFBQcDAjANBgkqhkiG9w0BAQsFAANBAAujxas6toRhMl/+kZrly0G/ AvjbA3WY5cLLIdffGdQ5bsS3aOP23nj98ut7unNUNsCo+eUwpJgvabnFnL+NFZA= -----END CERTIFICATE----- ================================================ FILE: nsqd/test/certs/client.req ================================================ -----BEGIN CERTIFICATE REQUEST----- MIICzTCCAbUCAQAwgYcxIzAhBgkqhkiG9w0BCQEWFG1yZWlmZXJzb25AZ21haWwu Y29tMQswCQYDVQQGEwJERTEMMAoGA1UECBMDTlJXMQ4wDAYDVQQHEwVFYXJ0aDEX MBUGA1UEChMOUmFuZG9tIENvbXBhbnkxCzAJBgNVBAsTAklUMQ8wDQYDVQQDEwZu c3EuaW8wggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQC4zPkgnTVapqPa okBTujZqhzaI/UFGFNtr5xauyNtkD7qRcCTX0OXP+c0ryDGwEyXevNMZZcpSb+so EOnrzTMmSPyfNwYCkaXNsOipfKbyLE6eI4Jof9E4M8Lw60PZSjN5eVQ5BYr65+Qy +ZUFz8h4nIWRybuOFf4sdUWf7eJYtE1vMaaFt/yg0e/JtLsdK5Xc5iIcMnEDR3oi CjOpU5hVKp0NptDYU3/ycrOkwePAyl9PTTzmFSGvvYTle4nduI9TCg6tERNJrX8R BwAQfTBLAdTUAZ2caHf/nKdW6l95/SqTKrA2x5iz1T74hQVeoDUi4SYCUmflBzMA eM9B7zEFAgMBAAGgADANBgkqhkiG9w0BAQsFAAOCAQEALD1VjbBjaeX7l7JR4IuL wrHVrFDiRYsgvqyw39j2MC95VwrGwzf4cXCE+RuqE/DhbV9UI7sWKaJfFs9Usq+g VSoKnHSEylt34y6ABSc5eAik+GnheJZbJ6UDjxcvNd0UpFGMrHbsXVyQd0Y1XAu7 nxCYIa82kNA+Opb+ra03hkLC7wvRLbXTOB2g6JLkyhYR6S5GkOFTNnz2AJerN7zt NEL7owlRcjyIsL5tjDpDH1944NNtzhgmrUeIjB08reayuot9RKznMVGwBfY6DIHM Q4uNN3CMOOoAHr1UzvBf/qfvb6ltPTMKSV1OncLlC3C59NoO8vhKIHCN18Ya2OMu rw== -----END CERTIFICATE REQUEST----- ================================================ FILE: nsqd/test/certs/key.pem ================================================ -----BEGIN RSA PRIVATE KEY----- MIIEogIBAAKCAQEAnX0KB+svwy+yHU2qggz/EaGgcraKShagKo+9M9y5HLM852ng k5c+t+tJJbx3N954Wr1FXBuGIv1ltU05rU4zhvBS25tVP1UIEnT5pBt2TeetLkl1 99Y7fxh1hKmnwJMG3fy3VZdNXEndBombXMmtXpQYshuEJHKeUNDbQKz5X+GjEdkT PO/HY/VMHsxS23pbSimQozMg3hvLIdgv0aS3QECzydZBgTPThy3uDtHIuCpxCwXd /vDF68ATlYgo3h3lh2vxNwM/pjklIUhzMh4XaKQF7m3/0KbtUcXfy0QHueeuMr11 E9MAFNyRN4xf9Fk1yB97KJ3PJBTC5WD/m1nW+QIDAQABAoIBACvtfKbIywG+hAf4 ad7skRjx5DcbA2e29+XnQfb9UgTXWd2SgrmoLi5OypBkCTzkKN3mfTo70yZfV8dC Sxwz+9tfnTz0DssjhKThS+CiaFVCkeOfSfBfKSlCQUVHrSrh18CDhP+yvDlJwQTZ zSQMfPcsh9bmJe2kqtQP7ZgUp1o+vaB8Sju8YYrO6FllxbdLRGm4pfvvrHIRRmXa oVHn0ei0JpwoTY9kHYht4LNeJnbP/MCWdmcuv3Gnel7jAlhaKab5aNIGr0Xe7aIQ iX6mpZ0/Rnt8o/XcTOg8l3ruIdVuySX6SYn08JMnfFkXdNYRVhoV1tC5ElWkaZLf hPmj2yECgYEAyts0R0b8cZ6HTAyuLm3ilw0s0v0/MM9ZtaqMRilr2WEtAhF0GpHG TzmGnii0WcTNXD7NTsNcECR/0ZpXPRleMczsL2Juwd4FkQ37h7hdKPseJNrfyHRg VolOFBX9H14C3wMB9cwdsG4Egw7fE27WCoreEquHgwFxl1zBrXKH088CgYEAxr8w BKZs0bF7LRrFT5pH8hpMLYHMYk8ZIOfgmEGVBKDQCOERPR9a9kqUss7wl/98LVNK RnFlyWD6Z0/QcQsLL4LjBeZJ25qEMc6JXm9VGAzhXA1ZkUofVoYCnG+f6KUn8CuJ /AcV2ZDFsEP10IiQG0hKsceXiwFEvEr8306tMrcCgYBLgnscSR0xAeyk71dq6vZc ecgEpcX+2kAvclOSzlpZ6WVCjtKkDT0/Qk+M0eQIQkybGLl9pxS+4Yc+s2/jy2yX pwsHvGE0AvwZeZX2eDcdSRR4bYy9ZixyKdwJeAHnyivRbaIuJ5Opl9pQGpoI9snv 1K9DTdw8dK4exKVHdgl/WwKBgDkmLsuXg4EEtPOyV/xc08VVNIR9Z2T5c7NXmeiO KyiKiWeUOF3ID2L07S9BfENozq9F3PzGjMtMXJSqibiHwW6nB1rh7mj8VHjx9+Q0 xVZGFeNfX1r84mgB3uxW2LeQDhzsmB/lda37CC14TU3qhu2hawEV8IijE73FHlOk Dv+fAoGAI4/XO5o5tNn5Djo8gHmGMCbinUE9+VySxl7wd7PK8w2VSofO88ofixDk NX94yBYhg5WZcLdPm45RyUnq+WVQYz9IKUrdxLFTH+wxyzUqZCW7jgXCvWV+071q vqm9C+kndq+18/1VKuCSGWnF7Ay4lbsgPXY2s4VKRxcb3QpZSPU= -----END RSA PRIVATE KEY----- ================================================ FILE: nsqd/test/certs/server.key ================================================ -----BEGIN RSA PRIVATE KEY----- MIIEowIBAAKCAQEAuYs7NerYpXWq/IPaBGriHobhIs/i4AJHra7QhZ5N/EfBCb0N 6K3tI4GRHMZ4lB0HwINWmxaPs6D2yDCxpGgnru6qCu0/91+Lq+GEZj+Fz1E2yfCq tecTySvWf9en+SdTKFZTGGIAu3zFcozuX2fdhkjJ6ibZhnrXKlPU4IN0STkkRfFX +yb/6P88mGh7PZKBb67Qw1bppzXE/SWgHVqDqu++whT1quwxnRhh/SHgwugN+6iK uTDcJ1jQ1QWe+xna7Te6nNBUvL6dV49jpoDOjHsv+UaAvzNeHCHqypTC0wwYHXyg TTAU+E20c2NOCC7P0PAid1liNeuuEWtM2bRrFQIDAQABAoIBABPzc7d1fDw2bd9f Mic9cvkDWdwLbILX2+tCG+vyPMJ+2LP6Xy+A3DnwKbFlafvLL1U1Ci/8+hC/oymd isx54qJ9yU0Je9JWtMcTpc/0zqefPPvz4/dRVKBSFWuDve0dnGR++8poZ1nBrd2G Z+9cVMamtwd1i/hY5yAHCaHmoK9qxXuSjtbPkqDqruloGO5iT+tcICjAeWyJIx1P 5VJjuKDx4AhU4B8DpBSTtA02BKltvr2K6D4zhJ255oI4iLUkYk4IwxB7yZH+6Y++ mDa6e8iM3ut+qMkCxto/CfMCqYfoRzmykiazZ5qFXfGwv3OkWUEOBXU//Vpklvnh OQWYL/UCgYEA7mkKXbCrCNJEdly4LVLPKPFjVUPhMdXcqdnoKZ6BiVYFMqTuG4+w o/Rf6ZpqZ3Rf656/ypd1atVvK2g4zfq3j1W6lDnpLjKaLhc8a+ZH4NWXBORIzlUW aR2xVcPpAokor37FRAWbduiwVSCHlDtg48i+rvukyqmeRBKjrIgX3lsCgYEAxzut 5GiMZ9TkjqYXdlUDIDKWMHMfLfRY7+Q/cBdr7AfiMVw4PIWJG/IuxCGwGp77gRg6 BeBsi8Htj5/EGbRiK4kjFyf8LqYGOeYLZ3I8+olr+tNmF4/yKwxku3VylpmrpCAV /8emso1rAWo6Y98MTNSGgvLqhDU6OD7tTSv3908CgYA4+aVeio/1RbrSxonFWxri 3/0rLVOuAzv+43KWL6kpVwNa/QtiTs6aABbDzwFKxAcAWinfkp6e727n4rpgj2A6 wvQZ5FUTk0hBZ5ArAReAZcr3gk7b8H2wlUYCBxWyY3Dzr8oY3XYvzqAFWAbOp/oZ tanMS5swS6TlA8dVvhhmLQKBgGZwcBPOEctdcntKOSwVv/qxJ/oXZ0O4rHYENP4M fOgqkYnxsdSkkH/3AUbFT4gQkJ6q90KIRyeA+gXsDudskUFzTMCeRZMyuGbSurBg 06u6NvQL+CVLVSf/QlgEpnt63f8QpF8Up8iM4CUlGoq5Z9ilOdhg0GZT+/BpopgY cHIPAoGBAKyLumgeG+gMFK5x/Fi6zjBT5MK8Tw0VMkahW01Jx2laYibS/a8AEFmn ySdrmLPkOmmWgXCk3m2m5PkvM5qH/KNugOA0+WX2CTvwt4ZgwWUalQ5R43wSIeDC MXVfwC8uE66PmpYgmsu4H0vnCGfacOCQhfdq01SLobgBiQSrm/6D -----END RSA PRIVATE KEY----- ================================================ FILE: nsqd/test/certs/server.pem ================================================ -----BEGIN CERTIFICATE----- MIIC3jCCAoigAwIBAgIJAJFBjQSZWSLmMA0GCSqGSIb3DQEBCwUAMA0xCzAJBgNV BAMMAmNhMB4XDTE3MDkxNjE3NDQxNFoXDTI3MDkxNDE3NDQxNFowgYcxIzAhBgkq hkiG9w0BCQEWFG1yZWlmZXJzb25AZ21haWwuY29tMQswCQYDVQQGEwJERTEMMAoG A1UECBMDTlJXMQ4wDAYDVQQHEwVFYXJ0aDEXMBUGA1UEChMOUmFuZG9tIENvbXBh bnkxCzAJBgNVBAsTAklUMQ8wDQYDVQQDEwZuc3EuaW8wggEiMA0GCSqGSIb3DQEB AQUAA4IBDwAwggEKAoIBAQC5izs16tildar8g9oEauIehuEiz+LgAketrtCFnk38 R8EJvQ3ore0jgZEcxniUHQfAg1abFo+zoPbIMLGkaCeu7qoK7T/3X4ur4YRmP4XP UTbJ8Kq15xPJK9Z/16f5J1MoVlMYYgC7fMVyjO5fZ92GSMnqJtmGetcqU9Tgg3RJ OSRF8Vf7Jv/o/zyYaHs9koFvrtDDVumnNcT9JaAdWoOq777CFPWq7DGdGGH9IeDC 6A37qIq5MNwnWNDVBZ77GdrtN7qc0FS8vp1Xj2OmgM6Mey/5RoC/M14cIerKlMLT DBgdfKBNMBT4TbRzY04ILs/Q8CJ3WWI1664Ra0zZtGsVAgMBAAGjgYcwgYQwDAYD VR0TAQH/BAIwADAdBgNVHQ4EFgQUt6UXJ8BQB++I/cQG5ZgaiUyqOE8wHwYDVR0j BBgwFoAUzIvHuuMoF7/FlAudXMi6o01XPFEwDgYDVR0PAQH/BAQDAgWgMBMGA1Ud JQQMMAoGCCsGAQUFBwMBMA8GA1UdEQQIMAaHBH8AAAEwDQYJKoZIhvcNAQELBQAD QQCCwm0F2eTmtIcKDxrXGZ7q9y9mfsROfYaCnH+vKUDw2vmqQkInzhLeEElDGQcR ww0IKCnDHEruNb2tKyQM/70L -----END CERTIFICATE----- ================================================ FILE: nsqd/test/certs/server.req ================================================ -----BEGIN CERTIFICATE REQUEST----- MIICzTCCAbUCAQAwgYcxIzAhBgkqhkiG9w0BCQEWFG1yZWlmZXJzb25AZ21haWwu Y29tMQswCQYDVQQGEwJERTEMMAoGA1UECBMDTlJXMQ4wDAYDVQQHEwVFYXJ0aDEX MBUGA1UEChMOUmFuZG9tIENvbXBhbnkxCzAJBgNVBAsTAklUMQ8wDQYDVQQDEwZu c3EuaW8wggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQC5izs16tildar8 g9oEauIehuEiz+LgAketrtCFnk38R8EJvQ3ore0jgZEcxniUHQfAg1abFo+zoPbI MLGkaCeu7qoK7T/3X4ur4YRmP4XPUTbJ8Kq15xPJK9Z/16f5J1MoVlMYYgC7fMVy jO5fZ92GSMnqJtmGetcqU9Tgg3RJOSRF8Vf7Jv/o/zyYaHs9koFvrtDDVumnNcT9 JaAdWoOq777CFPWq7DGdGGH9IeDC6A37qIq5MNwnWNDVBZ77GdrtN7qc0FS8vp1X j2OmgM6Mey/5RoC/M14cIerKlMLTDBgdfKBNMBT4TbRzY04ILs/Q8CJ3WWI1664R a0zZtGsVAgMBAAGgADANBgkqhkiG9w0BAQsFAAOCAQEASNjcoZbyNcwMQjMmNoil S/7pCRn4aYzZIVjrVtOHQ9GHC23MSep5um2gIcMFPiuYyu9Byl8CSVtc1op2fAKS vrugoZaCrp/A76hqOfNxgh7VmgTux8bG5Qcjaija1BNWpbyaWARdBxN/WgS5CpCj u2yzv8mrzzFNrDMlsmiEMvtkMzdhiZ4YY8zm6CdrbIR5z1eqf4e+rs4oJtTKNNAD hewk8CGiUW1hOx2jpjcIVMRy+ofVHRX2xQ6Sw8qxCNsiv8IPAAivgAbFJO76ZSbH eQ7uKWszmBEroyFvZ0rfmFLXuopU125pyBDl5FUKYAZzCBx9tr5dROCbw/rXDhke ig== -----END CERTIFICATE REQUEST----- ================================================ FILE: nsqd/test/openssl.conf ================================================ [req] distinguished_name = req_distinguished_name [req_distinguished_name] [ca] basicConstraints = critical, CA:true subjectKeyIdentifier = hash keyUsage = critical, cRLSign, keyCertSign [client] basicConstraints = critical, CA:FALSE subjectKeyIdentifier = hash authorityKeyIdentifier = keyid,issuer keyUsage = critical, digitalSignature, keyEncipherment extendedKeyUsage = clientAuth [server] basicConstraints = critical, CA:FALSE subjectKeyIdentifier = hash authorityKeyIdentifier = keyid,issuer keyUsage = critical, digitalSignature, keyEncipherment extendedKeyUsage = serverAuth subjectAltName = IP:127.0.0.1 ================================================ FILE: nsqd/topic.go ================================================ package nsqd import ( "errors" "strings" "sync" "sync/atomic" "time" "github.com/nsqio/go-diskqueue" "github.com/nsqio/nsq/internal/lg" "github.com/nsqio/nsq/internal/quantile" "github.com/nsqio/nsq/internal/util" ) type Topic struct { // 64bit atomic vars need to be first for proper alignment on 32bit platforms messageCount uint64 messageBytes uint64 sync.RWMutex name string channelMap map[string]*Channel backend BackendQueue memoryMsgChan chan *Message startChan chan int exitChan chan int channelUpdateChan chan int waitGroup util.WaitGroupWrapper exitFlag int32 idFactory *guidFactory ephemeral bool deleteCallback func(*Topic) deleter sync.Once paused int32 pauseChan chan int nsqd *NSQD } // Topic constructor func NewTopic(topicName string, nsqd *NSQD, deleteCallback func(*Topic)) *Topic { t := &Topic{ name: topicName, channelMap: make(map[string]*Channel), memoryMsgChan: make(chan *Message, nsqd.getOpts().MemQueueSize), startChan: make(chan int, 1), exitChan: make(chan int), channelUpdateChan: make(chan int), nsqd: nsqd, paused: 0, pauseChan: make(chan int), deleteCallback: deleteCallback, idFactory: NewGUIDFactory(nsqd.getOpts().ID), } if strings.HasSuffix(topicName, "#ephemeral") { t.ephemeral = true t.backend = newDummyBackendQueue() } else { dqLogf := func(level diskqueue.LogLevel, f string, args ...interface{}) { opts := nsqd.getOpts() lg.Logf(opts.Logger, opts.LogLevel, lg.LogLevel(level), f, args...) } t.backend = diskqueue.New( topicName, nsqd.getOpts().DataPath, nsqd.getOpts().MaxBytesPerFile, int32(minValidMsgLength), int32(nsqd.getOpts().MaxMsgSize)+minValidMsgLength, nsqd.getOpts().SyncEvery, nsqd.getOpts().SyncTimeout, dqLogf, ) } t.waitGroup.Wrap(t.messagePump) t.nsqd.Notify(t, !t.ephemeral) return t } func (t *Topic) Start() { select { case t.startChan <- 1: default: } } // Exiting returns a boolean indicating if this topic is closed/exiting func (t *Topic) Exiting() bool { return atomic.LoadInt32(&t.exitFlag) == 1 } // GetChannel performs a thread safe operation // to return a pointer to a Channel object (potentially new) // for the given Topic func (t *Topic) GetChannel(channelName string) *Channel { t.Lock() channel, isNew := t.getOrCreateChannel(channelName) t.Unlock() if isNew { // update messagePump state select { case t.channelUpdateChan <- 1: case <-t.exitChan: } } return channel } // this expects the caller to handle locking func (t *Topic) getOrCreateChannel(channelName string) (*Channel, bool) { channel, ok := t.channelMap[channelName] if !ok { deleteCallback := func(c *Channel) { t.DeleteExistingChannel(c.name) } channel = NewChannel(t.name, channelName, t.nsqd, deleteCallback) t.channelMap[channelName] = channel t.nsqd.logf(LOG_INFO, "TOPIC(%s): new channel(%s)", t.name, channel.name) return channel, true } return channel, false } func (t *Topic) GetExistingChannel(channelName string) (*Channel, error) { t.RLock() defer t.RUnlock() channel, ok := t.channelMap[channelName] if !ok { return nil, errors.New("channel does not exist") } return channel, nil } // DeleteExistingChannel removes a channel from the topic only if it exists func (t *Topic) DeleteExistingChannel(channelName string) error { t.RLock() channel, ok := t.channelMap[channelName] t.RUnlock() if !ok { return errors.New("channel does not exist") } t.nsqd.logf(LOG_INFO, "TOPIC(%s): deleting channel %s", t.name, channel.name) // delete empties the channel before closing // (so that we dont leave any messages around) // // we do this before removing the channel from map below (with no lock) // so that any incoming subs will error and not create a new channel // to enforce ordering channel.Delete() t.Lock() delete(t.channelMap, channelName) numChannels := len(t.channelMap) t.Unlock() // update messagePump state select { case t.channelUpdateChan <- 1: case <-t.exitChan: } if numChannels == 0 && t.ephemeral { go t.deleter.Do(func() { t.deleteCallback(t) }) } return nil } // PutMessage writes a Message to the queue func (t *Topic) PutMessage(m *Message) error { t.RLock() defer t.RUnlock() if atomic.LoadInt32(&t.exitFlag) == 1 { return errors.New("exiting") } err := t.put(m) if err != nil { return err } atomic.AddUint64(&t.messageCount, 1) atomic.AddUint64(&t.messageBytes, uint64(len(m.Body))) return nil } // PutMessages writes multiple Messages to the queue func (t *Topic) PutMessages(msgs []*Message) error { t.RLock() defer t.RUnlock() if atomic.LoadInt32(&t.exitFlag) == 1 { return errors.New("exiting") } messageTotalBytes := 0 for i, m := range msgs { err := t.put(m) if err != nil { atomic.AddUint64(&t.messageCount, uint64(i)) atomic.AddUint64(&t.messageBytes, uint64(messageTotalBytes)) return err } messageTotalBytes += len(m.Body) } atomic.AddUint64(&t.messageBytes, uint64(messageTotalBytes)) atomic.AddUint64(&t.messageCount, uint64(len(msgs))) return nil } func (t *Topic) put(m *Message) error { // If mem-queue-size == 0, avoid memory chan, for more consistent ordering, // but try to use memory chan for deferred messages (they lose deferred timer // in backend queue) or if topic is ephemeral (there is no backend queue). if cap(t.memoryMsgChan) > 0 || t.ephemeral || m.deferred != 0 { select { case t.memoryMsgChan <- m: return nil default: break // write to backend } } err := writeMessageToBackend(m, t.backend) t.nsqd.SetHealth(err) if err != nil { t.nsqd.logf(LOG_ERROR, "TOPIC(%s) ERROR: failed to write message to backend - %s", t.name, err) return err } return nil } func (t *Topic) Depth() int64 { return int64(len(t.memoryMsgChan)) + t.backend.Depth() } // messagePump selects over the in-memory and backend queue and // writes messages to every channel for this topic func (t *Topic) messagePump() { var msg *Message var buf []byte var err error var chans []*Channel var memoryMsgChan chan *Message var backendChan <-chan []byte // do not pass messages before Start(), but avoid blocking Pause() or GetChannel() for { select { case <-t.channelUpdateChan: continue case <-t.pauseChan: continue case <-t.exitChan: goto exit case <-t.startChan: } break } t.RLock() for _, c := range t.channelMap { chans = append(chans, c) } t.RUnlock() if len(chans) > 0 && !t.IsPaused() { memoryMsgChan = t.memoryMsgChan backendChan = t.backend.ReadChan() } // main message loop for { select { case msg = <-memoryMsgChan: case buf = <-backendChan: msg, err = decodeMessage(buf) if err != nil { t.nsqd.logf(LOG_ERROR, "failed to decode message - %s", err) continue } case <-t.channelUpdateChan: chans = chans[:0] t.RLock() for _, c := range t.channelMap { chans = append(chans, c) } t.RUnlock() if len(chans) == 0 || t.IsPaused() { memoryMsgChan = nil backendChan = nil } else { memoryMsgChan = t.memoryMsgChan backendChan = t.backend.ReadChan() } continue case <-t.pauseChan: if len(chans) == 0 || t.IsPaused() { memoryMsgChan = nil backendChan = nil } else { memoryMsgChan = t.memoryMsgChan backendChan = t.backend.ReadChan() } continue case <-t.exitChan: goto exit } for i, channel := range chans { chanMsg := msg // copy the message because each channel // needs a unique instance but... // fastpath to avoid copy if its the first channel // (the topic already created the first copy) if i > 0 { chanMsg = NewMessage(msg.ID, msg.Body) chanMsg.Timestamp = msg.Timestamp chanMsg.deferred = msg.deferred } if chanMsg.deferred != 0 { channel.PutMessageDeferred(chanMsg, chanMsg.deferred) continue } err := channel.PutMessage(chanMsg) if err != nil { t.nsqd.logf(LOG_ERROR, "TOPIC(%s) ERROR: failed to put msg(%s) to channel(%s) - %s", t.name, msg.ID, channel.name, err) } } } exit: t.nsqd.logf(LOG_INFO, "TOPIC(%s): closing ... messagePump", t.name) } // Delete empties the topic and all its channels and closes func (t *Topic) Delete() error { return t.exit(true) } // Close persists all outstanding topic data and closes all its channels func (t *Topic) Close() error { return t.exit(false) } func (t *Topic) exit(deleted bool) error { if !atomic.CompareAndSwapInt32(&t.exitFlag, 0, 1) { return errors.New("exiting") } if deleted { t.nsqd.logf(LOG_INFO, "TOPIC(%s): deleting", t.name) // since we are explicitly deleting a topic (not just at system exit time) // de-register this from the lookupd t.nsqd.Notify(t, !t.ephemeral) } else { t.nsqd.logf(LOG_INFO, "TOPIC(%s): closing", t.name) } close(t.exitChan) // synchronize the close of messagePump() t.waitGroup.Wait() if deleted { t.Lock() for _, channel := range t.channelMap { delete(t.channelMap, channel.name) channel.Delete() } t.Unlock() // empty the queue (deletes the backend files, too) t.Empty() return t.backend.Delete() } // close all the channels t.RLock() for _, channel := range t.channelMap { err := channel.Close() if err != nil { // we need to continue regardless of error to close all the channels t.nsqd.logf(LOG_ERROR, "channel(%s) close - %s", channel.name, err) } } t.RUnlock() // write anything leftover to disk t.flush() return t.backend.Close() } func (t *Topic) Empty() error { for { select { case <-t.memoryMsgChan: default: goto finish } } finish: return t.backend.Empty() } func (t *Topic) flush() error { if len(t.memoryMsgChan) > 0 { t.nsqd.logf(LOG_INFO, "TOPIC(%s): flushing %d memory messages to backend", t.name, len(t.memoryMsgChan)) } for { select { case msg := <-t.memoryMsgChan: err := writeMessageToBackend(msg, t.backend) if err != nil { t.nsqd.logf(LOG_ERROR, "ERROR: failed to write message to backend - %s", err) } default: goto finish } } finish: return nil } func (t *Topic) AggregateChannelE2eProcessingLatency() *quantile.Quantile { var latencyStream *quantile.Quantile t.RLock() realChannels := make([]*Channel, 0, len(t.channelMap)) for _, c := range t.channelMap { realChannels = append(realChannels, c) } t.RUnlock() for _, c := range realChannels { if c.e2eProcessingLatencyStream == nil { continue } if latencyStream == nil { latencyStream = quantile.New( t.nsqd.getOpts().E2EProcessingLatencyWindowTime, t.nsqd.getOpts().E2EProcessingLatencyPercentiles) } latencyStream.Merge(c.e2eProcessingLatencyStream) } return latencyStream } func (t *Topic) Pause() error { return t.doPause(true) } func (t *Topic) UnPause() error { return t.doPause(false) } func (t *Topic) doPause(pause bool) error { if pause { atomic.StoreInt32(&t.paused, 1) } else { atomic.StoreInt32(&t.paused, 0) } select { case t.pauseChan <- 1: case <-t.exitChan: } return nil } func (t *Topic) IsPaused() bool { return atomic.LoadInt32(&t.paused) == 1 } func (t *Topic) GenerateID() MessageID { var i int64 = 0 for { id, err := t.idFactory.NewGUID() if err == nil { return id.Hex() } if i%10000 == 0 { t.nsqd.logf(LOG_ERROR, "TOPIC(%s): failed to create guid - %s", t.name, err) } time.Sleep(time.Millisecond) i++ } } ================================================ FILE: nsqd/topic_test.go ================================================ package nsqd import ( "errors" "fmt" "io" "net/http" "os" "runtime" "strconv" "testing" "time" "github.com/nsqio/nsq/internal/test" ) func TestGetTopic(t *testing.T) { opts := NewOptions() opts.Logger = test.NewTestLogger(t) _, _, nsqd := mustStartNSQD(opts) defer os.RemoveAll(opts.DataPath) defer nsqd.Exit() topic1 := nsqd.GetTopic("test") test.NotNil(t, topic1) test.Equal(t, "test", topic1.name) topic2 := nsqd.GetTopic("test") test.Equal(t, topic1, topic2) topic3 := nsqd.GetTopic("test2") test.Equal(t, "test2", topic3.name) test.NotEqual(t, topic2, topic3) } func TestGetChannel(t *testing.T) { opts := NewOptions() opts.Logger = test.NewTestLogger(t) _, _, nsqd := mustStartNSQD(opts) defer os.RemoveAll(opts.DataPath) defer nsqd.Exit() topic := nsqd.GetTopic("test") channel1 := topic.GetChannel("ch1") test.NotNil(t, channel1) test.Equal(t, "ch1", channel1.name) channel2 := topic.GetChannel("ch2") test.Equal(t, channel1, topic.channelMap["ch1"]) test.Equal(t, channel2, topic.channelMap["ch2"]) } type errorBackendQueue struct{} func (d *errorBackendQueue) Put([]byte) error { return errors.New("never gonna happen") } func (d *errorBackendQueue) ReadChan() <-chan []byte { return nil } func (d *errorBackendQueue) Close() error { return nil } func (d *errorBackendQueue) Delete() error { return nil } func (d *errorBackendQueue) Depth() int64 { return 0 } func (d *errorBackendQueue) Empty() error { return nil } type errorRecoveredBackendQueue struct{ errorBackendQueue } func (d *errorRecoveredBackendQueue) Put([]byte) error { return nil } func TestHealth(t *testing.T) { opts := NewOptions() opts.Logger = test.NewTestLogger(t) opts.MemQueueSize = 2 _, httpAddr, nsqd := mustStartNSQD(opts) defer os.RemoveAll(opts.DataPath) defer nsqd.Exit() topic := nsqd.GetTopic("test") topic.backend = &errorBackendQueue{} msg := NewMessage(topic.GenerateID(), make([]byte, 100)) err := topic.PutMessage(msg) test.Nil(t, err) msg = NewMessage(topic.GenerateID(), make([]byte, 100)) err = topic.PutMessages([]*Message{msg}) test.Nil(t, err) msg = NewMessage(topic.GenerateID(), make([]byte, 100)) err = topic.PutMessage(msg) test.NotNil(t, err) msg = NewMessage(topic.GenerateID(), make([]byte, 100)) err = topic.PutMessages([]*Message{msg}) test.NotNil(t, err) url := fmt.Sprintf("http://%s/ping", httpAddr) resp, err := http.Get(url) test.Nil(t, err) test.Equal(t, 500, resp.StatusCode) body, _ := io.ReadAll(resp.Body) resp.Body.Close() test.Equal(t, "NOK - never gonna happen", string(body)) topic.backend = &errorRecoveredBackendQueue{} msg = NewMessage(topic.GenerateID(), make([]byte, 100)) err = topic.PutMessages([]*Message{msg}) test.Nil(t, err) resp, err = http.Get(url) test.Nil(t, err) test.Equal(t, 200, resp.StatusCode) body, _ = io.ReadAll(resp.Body) resp.Body.Close() test.Equal(t, "OK", string(body)) } func TestDeletes(t *testing.T) { opts := NewOptions() opts.Logger = test.NewTestLogger(t) _, _, nsqd := mustStartNSQD(opts) defer os.RemoveAll(opts.DataPath) defer nsqd.Exit() topic := nsqd.GetTopic("test") channel1 := topic.GetChannel("ch1") test.NotNil(t, channel1) err := topic.DeleteExistingChannel("ch1") test.Nil(t, err) test.Equal(t, 0, len(topic.channelMap)) channel2 := topic.GetChannel("ch2") test.NotNil(t, channel2) err = nsqd.DeleteExistingTopic("test") test.Nil(t, err) test.Equal(t, 0, len(topic.channelMap)) test.Equal(t, 0, len(nsqd.topicMap)) } func TestDeleteLast(t *testing.T) { opts := NewOptions() opts.Logger = test.NewTestLogger(t) _, _, nsqd := mustStartNSQD(opts) defer os.RemoveAll(opts.DataPath) defer nsqd.Exit() topic := nsqd.GetTopic("test") channel1 := topic.GetChannel("ch1") test.NotNil(t, channel1) err := topic.DeleteExistingChannel("ch1") test.Nil(t, err) test.Equal(t, 0, len(topic.channelMap)) msg := NewMessage(topic.GenerateID(), []byte("aaaaaaaaaaaaaaaaaaaaaaaaaaa")) err = topic.PutMessage(msg) time.Sleep(100 * time.Millisecond) test.Nil(t, err) test.Equal(t, int64(1), topic.Depth()) } func TestPause(t *testing.T) { opts := NewOptions() opts.Logger = test.NewTestLogger(t) _, _, nsqd := mustStartNSQD(opts) defer os.RemoveAll(opts.DataPath) defer nsqd.Exit() topicName := "test_topic_pause" + strconv.Itoa(int(time.Now().Unix())) topic := nsqd.GetTopic(topicName) err := topic.Pause() test.Nil(t, err) channel := topic.GetChannel("ch1") test.NotNil(t, channel) msg := NewMessage(topic.GenerateID(), []byte("aaaaaaaaaaaaaaaaaaaaaaaaaaa")) err = topic.PutMessage(msg) test.Nil(t, err) time.Sleep(15 * time.Millisecond) test.Equal(t, int64(1), topic.Depth()) test.Equal(t, int64(0), channel.Depth()) err = topic.UnPause() test.Nil(t, err) time.Sleep(15 * time.Millisecond) test.Equal(t, int64(0), topic.Depth()) test.Equal(t, int64(1), channel.Depth()) } func BenchmarkTopicPut(b *testing.B) { b.StopTimer() topicName := "bench_topic_put" + strconv.Itoa(b.N) opts := NewOptions() opts.Logger = test.NewTestLogger(b) opts.MemQueueSize = int64(b.N) _, _, nsqd := mustStartNSQD(opts) defer os.RemoveAll(opts.DataPath) defer nsqd.Exit() b.StartTimer() for i := 0; i <= b.N; i++ { topic := nsqd.GetTopic(topicName) msg := NewMessage(topic.GenerateID(), []byte("aaaaaaaaaaaaaaaaaaaaaaaaaaa")) topic.PutMessage(msg) } } func BenchmarkTopicToChannelPut(b *testing.B) { b.StopTimer() topicName := "bench_topic_to_channel_put" + strconv.Itoa(b.N) channelName := "bench" opts := NewOptions() opts.Logger = test.NewTestLogger(b) opts.MemQueueSize = int64(b.N) _, _, nsqd := mustStartNSQD(opts) defer os.RemoveAll(opts.DataPath) defer nsqd.Exit() channel := nsqd.GetTopic(topicName).GetChannel(channelName) b.StartTimer() for i := 0; i <= b.N; i++ { topic := nsqd.GetTopic(topicName) msg := NewMessage(topic.GenerateID(), []byte("aaaaaaaaaaaaaaaaaaaaaaaaaaa")) topic.PutMessage(msg) } for { if len(channel.memoryMsgChan) == b.N { break } runtime.Gosched() } } ================================================ FILE: nsqlookupd/README.md ================================================ ## nsqlookupd `nsqlookupd` is the daemon that manages topology metadata and serves client requests to discover the location of topics at runtime. Read the [docs](https://nsq.io/components/nsqlookupd.html) ================================================ FILE: nsqlookupd/client_v1.go ================================================ package nsqlookupd import ( "net" ) type ClientV1 struct { net.Conn peerInfo *PeerInfo } func NewClientV1(conn net.Conn) *ClientV1 { return &ClientV1{ Conn: conn, } } func (c *ClientV1) String() string { return c.RemoteAddr().String() } ================================================ FILE: nsqlookupd/http.go ================================================ package nsqlookupd import ( "fmt" "net/http" "net/http/pprof" "sync/atomic" "github.com/julienschmidt/httprouter" "github.com/nsqio/nsq/internal/http_api" "github.com/nsqio/nsq/internal/protocol" "github.com/nsqio/nsq/internal/version" ) type httpServer struct { nsqlookupd *NSQLookupd router http.Handler } func newHTTPServer(l *NSQLookupd) *httpServer { log := http_api.Log(l.logf) router := httprouter.New() router.HandleMethodNotAllowed = true router.PanicHandler = http_api.LogPanicHandler(l.logf) router.NotFound = http_api.LogNotFoundHandler(l.logf) router.MethodNotAllowed = http_api.LogMethodNotAllowedHandler(l.logf) s := &httpServer{ nsqlookupd: l, router: router, } router.Handle("GET", "/ping", http_api.Decorate(s.pingHandler, log, http_api.PlainText)) router.Handle("GET", "/info", http_api.Decorate(s.doInfo, log, http_api.V1)) // v1 negotiate router.Handle("GET", "/debug", http_api.Decorate(s.doDebug, log, http_api.V1)) router.Handle("GET", "/lookup", http_api.Decorate(s.doLookup, log, http_api.V1)) router.Handle("GET", "/topics", http_api.Decorate(s.doTopics, log, http_api.V1)) router.Handle("GET", "/channels", http_api.Decorate(s.doChannels, log, http_api.V1)) router.Handle("GET", "/nodes", http_api.Decorate(s.doNodes, log, http_api.V1)) // only v1 router.Handle("POST", "/topic/create", http_api.Decorate(s.doCreateTopic, log, http_api.V1)) router.Handle("POST", "/topic/delete", http_api.Decorate(s.doDeleteTopic, log, http_api.V1)) router.Handle("POST", "/channel/create", http_api.Decorate(s.doCreateChannel, log, http_api.V1)) router.Handle("POST", "/channel/delete", http_api.Decorate(s.doDeleteChannel, log, http_api.V1)) router.Handle("POST", "/topic/tombstone", http_api.Decorate(s.doTombstoneTopicProducer, log, http_api.V1)) // debug router.HandlerFunc("GET", "/debug/pprof", pprof.Index) router.HandlerFunc("GET", "/debug/pprof/cmdline", pprof.Cmdline) router.HandlerFunc("GET", "/debug/pprof/symbol", pprof.Symbol) router.HandlerFunc("POST", "/debug/pprof/symbol", pprof.Symbol) router.HandlerFunc("GET", "/debug/pprof/profile", pprof.Profile) router.Handler("GET", "/debug/pprof/heap", pprof.Handler("heap")) router.Handler("GET", "/debug/pprof/goroutine", pprof.Handler("goroutine")) router.Handler("GET", "/debug/pprof/block", pprof.Handler("block")) router.Handler("GET", "/debug/pprof/threadcreate", pprof.Handler("threadcreate")) return s } func (s *httpServer) ServeHTTP(w http.ResponseWriter, req *http.Request) { s.router.ServeHTTP(w, req) } func (s *httpServer) pingHandler(w http.ResponseWriter, req *http.Request, ps httprouter.Params) (interface{}, error) { return "OK", nil } func (s *httpServer) doInfo(w http.ResponseWriter, req *http.Request, ps httprouter.Params) (interface{}, error) { return struct { Version string `json:"version"` }{ Version: version.Binary, }, nil } func (s *httpServer) doTopics(w http.ResponseWriter, req *http.Request, ps httprouter.Params) (interface{}, error) { topics := s.nsqlookupd.DB.FindRegistrations("topic", "*", "").Keys() return map[string]interface{}{ "topics": topics, }, nil } func (s *httpServer) doChannels(w http.ResponseWriter, req *http.Request, ps httprouter.Params) (interface{}, error) { reqParams, err := http_api.NewReqParams(req) if err != nil { return nil, http_api.Err{400, "INVALID_REQUEST"} } topicName, err := reqParams.Get("topic") if err != nil { return nil, http_api.Err{400, "MISSING_ARG_TOPIC"} } channels := s.nsqlookupd.DB.FindRegistrations("channel", topicName, "*").SubKeys() return map[string]interface{}{ "channels": channels, }, nil } func (s *httpServer) doLookup(w http.ResponseWriter, req *http.Request, ps httprouter.Params) (interface{}, error) { reqParams, err := http_api.NewReqParams(req) if err != nil { return nil, http_api.Err{400, "INVALID_REQUEST"} } topicName, err := reqParams.Get("topic") if err != nil { return nil, http_api.Err{400, "MISSING_ARG_TOPIC"} } registration := s.nsqlookupd.DB.FindRegistrations("topic", topicName, "") if len(registration) == 0 { return nil, http_api.Err{404, "TOPIC_NOT_FOUND"} } channels := s.nsqlookupd.DB.FindRegistrations("channel", topicName, "*").SubKeys() producers := s.nsqlookupd.DB.FindProducers("topic", topicName, "") producers = producers.FilterByActive(s.nsqlookupd.opts.InactiveProducerTimeout, s.nsqlookupd.opts.TombstoneLifetime) return map[string]interface{}{ "channels": channels, "producers": producers.PeerInfo(), }, nil } func (s *httpServer) doCreateTopic(w http.ResponseWriter, req *http.Request, ps httprouter.Params) (interface{}, error) { reqParams, err := http_api.NewReqParams(req) if err != nil { return nil, http_api.Err{400, "INVALID_REQUEST"} } topicName, err := reqParams.Get("topic") if err != nil { return nil, http_api.Err{400, "MISSING_ARG_TOPIC"} } if !protocol.IsValidTopicName(topicName) { return nil, http_api.Err{400, "INVALID_ARG_TOPIC"} } s.nsqlookupd.logf(LOG_INFO, "DB: adding topic(%s)", topicName) key := Registration{"topic", topicName, ""} s.nsqlookupd.DB.AddRegistration(key) return nil, nil } func (s *httpServer) doDeleteTopic(w http.ResponseWriter, req *http.Request, ps httprouter.Params) (interface{}, error) { reqParams, err := http_api.NewReqParams(req) if err != nil { return nil, http_api.Err{400, "INVALID_REQUEST"} } topicName, err := reqParams.Get("topic") if err != nil { return nil, http_api.Err{400, "MISSING_ARG_TOPIC"} } registrations := s.nsqlookupd.DB.FindRegistrations("channel", topicName, "*") for _, registration := range registrations { s.nsqlookupd.logf(LOG_INFO, "DB: removing channel(%s) from topic(%s)", registration.SubKey, topicName) s.nsqlookupd.DB.RemoveRegistration(registration) } registrations = s.nsqlookupd.DB.FindRegistrations("topic", topicName, "") for _, registration := range registrations { s.nsqlookupd.logf(LOG_INFO, "DB: removing topic(%s)", topicName) s.nsqlookupd.DB.RemoveRegistration(registration) } return nil, nil } func (s *httpServer) doTombstoneTopicProducer(w http.ResponseWriter, req *http.Request, ps httprouter.Params) (interface{}, error) { reqParams, err := http_api.NewReqParams(req) if err != nil { return nil, http_api.Err{400, "INVALID_REQUEST"} } topicName, err := reqParams.Get("topic") if err != nil { return nil, http_api.Err{400, "MISSING_ARG_TOPIC"} } node, err := reqParams.Get("node") if err != nil { return nil, http_api.Err{400, "MISSING_ARG_NODE"} } s.nsqlookupd.logf(LOG_INFO, "DB: setting tombstone for producer@%s of topic(%s)", node, topicName) producers := s.nsqlookupd.DB.FindProducers("topic", topicName, "") for _, p := range producers { thisNode := fmt.Sprintf("%s:%d", p.peerInfo.BroadcastAddress, p.peerInfo.HTTPPort) if thisNode == node { p.Tombstone() } } return nil, nil } func (s *httpServer) doCreateChannel(w http.ResponseWriter, req *http.Request, ps httprouter.Params) (interface{}, error) { reqParams, err := http_api.NewReqParams(req) if err != nil { return nil, http_api.Err{400, "INVALID_REQUEST"} } topicName, channelName, err := http_api.GetTopicChannelArgs(reqParams) if err != nil { return nil, http_api.Err{400, err.Error()} } s.nsqlookupd.logf(LOG_INFO, "DB: adding channel(%s) in topic(%s)", channelName, topicName) key := Registration{"channel", topicName, channelName} s.nsqlookupd.DB.AddRegistration(key) s.nsqlookupd.logf(LOG_INFO, "DB: adding topic(%s)", topicName) key = Registration{"topic", topicName, ""} s.nsqlookupd.DB.AddRegistration(key) return nil, nil } func (s *httpServer) doDeleteChannel(w http.ResponseWriter, req *http.Request, ps httprouter.Params) (interface{}, error) { reqParams, err := http_api.NewReqParams(req) if err != nil { return nil, http_api.Err{400, "INVALID_REQUEST"} } topicName, channelName, err := http_api.GetTopicChannelArgs(reqParams) if err != nil { return nil, http_api.Err{400, err.Error()} } registrations := s.nsqlookupd.DB.FindRegistrations("channel", topicName, channelName) if len(registrations) == 0 { return nil, http_api.Err{404, "CHANNEL_NOT_FOUND"} } s.nsqlookupd.logf(LOG_INFO, "DB: removing channel(%s) from topic(%s)", channelName, topicName) for _, registration := range registrations { s.nsqlookupd.DB.RemoveRegistration(registration) } return nil, nil } type node struct { RemoteAddress string `json:"remote_address"` Hostname string `json:"hostname"` BroadcastAddress string `json:"broadcast_address"` TCPPort int `json:"tcp_port"` HTTPPort int `json:"http_port"` Version string `json:"version"` ToplogyZone string `json:"topology_zone"` ToplogyRegion string `json:"topology_region"` Tombstones []bool `json:"tombstones"` Topics []string `json:"topics"` } func (s *httpServer) doNodes(w http.ResponseWriter, req *http.Request, ps httprouter.Params) (interface{}, error) { // dont filter out tombstoned nodes producers := s.nsqlookupd.DB.FindProducers("client", "", "").FilterByActive( s.nsqlookupd.opts.InactiveProducerTimeout, 0) nodes := make([]*node, len(producers)) topicProducersMap := make(map[string]Producers) for i, p := range producers { topics := s.nsqlookupd.DB.LookupRegistrations(p.peerInfo.id).Filter("topic", "*", "").Keys() // for each topic find the producer that matches this peer // to add tombstone information tombstones := make([]bool, len(topics)) for j, t := range topics { if _, exists := topicProducersMap[t]; !exists { topicProducersMap[t] = s.nsqlookupd.DB.FindProducers("topic", t, "") } topicProducers := topicProducersMap[t] for _, tp := range topicProducers { if tp.peerInfo == p.peerInfo { tombstones[j] = tp.IsTombstoned(s.nsqlookupd.opts.TombstoneLifetime) break } } } nodes[i] = &node{ RemoteAddress: p.peerInfo.RemoteAddress, Hostname: p.peerInfo.Hostname, BroadcastAddress: p.peerInfo.BroadcastAddress, TCPPort: p.peerInfo.TCPPort, HTTPPort: p.peerInfo.HTTPPort, Version: p.peerInfo.Version, ToplogyZone: p.peerInfo.TopologyZone, ToplogyRegion: p.peerInfo.TopologyRegion, Tombstones: tombstones, Topics: topics, } } return map[string]interface{}{ "producers": nodes, }, nil } func (s *httpServer) doDebug(w http.ResponseWriter, req *http.Request, ps httprouter.Params) (interface{}, error) { s.nsqlookupd.DB.RLock() defer s.nsqlookupd.DB.RUnlock() data := make(map[string][]map[string]interface{}) for r, producers := range s.nsqlookupd.DB.registrationMap { key := r.Category + ":" + r.Key + ":" + r.SubKey for _, p := range producers { m := map[string]interface{}{ "id": p.peerInfo.id, "hostname": p.peerInfo.Hostname, "broadcast_address": p.peerInfo.BroadcastAddress, "tcp_port": p.peerInfo.TCPPort, "http_port": p.peerInfo.HTTPPort, "version": p.peerInfo.Version, "topology_zone": p.peerInfo.TopologyZone, "topology_region": p.peerInfo.TopologyRegion, "last_update": atomic.LoadInt64(&p.peerInfo.lastUpdate), "tombstoned": p.tombstoned, "tombstoned_at": p.tombstonedAt.UnixNano(), } data[key] = append(data[key], m) } } return data, nil } ================================================ FILE: nsqlookupd/http_test.go ================================================ package nsqlookupd import ( "encoding/json" "fmt" "io" "net/http" "os" "strconv" "testing" "time" "github.com/nsqio/nsq/internal/test" "github.com/nsqio/nsq/internal/version" "github.com/nsqio/nsq/nsqd" ) type InfoDoc struct { Version string `json:"version"` } type ChannelsDoc struct { Channels []interface{} `json:"channels"` } type ErrMessage struct { Message string `json:"message"` } func bootstrapNSQCluster(t *testing.T) (string, []*nsqd.NSQD, *NSQLookupd) { lgr := test.NewTestLogger(t) nsqlookupdOpts := NewOptions() nsqlookupdOpts.TCPAddress = "127.0.0.1:0" nsqlookupdOpts.HTTPAddress = "127.0.0.1:0" nsqlookupdOpts.BroadcastAddress = "127.0.0.1" nsqlookupdOpts.Logger = lgr nsqlookupd1, err := New(nsqlookupdOpts) if err != nil { panic(err) } go func() { err := nsqlookupd1.Main() if err != nil { panic(err) } }() time.Sleep(100 * time.Millisecond) nsqdOpts := nsqd.NewOptions() nsqdOpts.TCPAddress = "127.0.0.1:0" nsqdOpts.HTTPAddress = "127.0.0.1:0" nsqdOpts.BroadcastAddress = "127.0.0.1" nsqdOpts.NSQLookupdTCPAddresses = []string{nsqlookupd1.RealTCPAddr().String()} nsqdOpts.Logger = lgr tmpDir, err := os.MkdirTemp("", fmt.Sprintf("nsq-test-%d", time.Now().UnixNano())) if err != nil { panic(err) } nsqdOpts.DataPath = tmpDir nsqd1, err := nsqd.New(nsqdOpts) if err != nil { panic(err) } go func() { err := nsqd1.Main() if err != nil { panic(err) } }() time.Sleep(100 * time.Millisecond) return tmpDir, []*nsqd.NSQD{nsqd1}, nsqlookupd1 } func makeTopic(nsqlookupd *NSQLookupd, topicName string) { key := Registration{"topic", topicName, ""} nsqlookupd.DB.AddRegistration(key) } func makeChannel(nsqlookupd *NSQLookupd, topicName string, channelName string) { key := Registration{"channel", topicName, channelName} nsqlookupd.DB.AddRegistration(key) makeTopic(nsqlookupd, topicName) } func TestPing(t *testing.T) { dataPath, nsqds, nsqlookupd1 := bootstrapNSQCluster(t) defer os.RemoveAll(dataPath) defer nsqds[0].Exit() defer nsqlookupd1.Exit() client := http.Client{} url := fmt.Sprintf("http://%s/ping", nsqlookupd1.RealHTTPAddr()) req, _ := http.NewRequest("GET", url, nil) resp, err := client.Do(req) test.Nil(t, err) test.Equal(t, 200, resp.StatusCode) body, _ := io.ReadAll(resp.Body) resp.Body.Close() test.Equal(t, []byte("OK"), body) } func TestInfo(t *testing.T) { dataPath, nsqds, nsqlookupd1 := bootstrapNSQCluster(t) defer os.RemoveAll(dataPath) defer nsqds[0].Exit() defer nsqlookupd1.Exit() client := http.Client{} url := fmt.Sprintf("http://%s/info", nsqlookupd1.RealHTTPAddr()) req, _ := http.NewRequest("GET", url, nil) resp, err := client.Do(req) test.Nil(t, err) test.Equal(t, 200, resp.StatusCode) body, _ := io.ReadAll(resp.Body) resp.Body.Close() t.Logf("%s", body) info := InfoDoc{} err = json.Unmarshal(body, &info) test.Nil(t, err) test.Equal(t, version.Binary, info.Version) } func TestCreateTopic(t *testing.T) { dataPath, nsqds, nsqlookupd1 := bootstrapNSQCluster(t) defer os.RemoveAll(dataPath) defer nsqds[0].Exit() defer nsqlookupd1.Exit() em := ErrMessage{} client := http.Client{} url := fmt.Sprintf("http://%s/topic/create", nsqlookupd1.RealHTTPAddr()) req, _ := http.NewRequest("POST", url, nil) resp, err := client.Do(req) test.Nil(t, err) test.Equal(t, 400, resp.StatusCode) test.Equal(t, "Bad Request", http.StatusText(resp.StatusCode)) body, _ := io.ReadAll(resp.Body) resp.Body.Close() t.Logf("%s", body) err = json.Unmarshal(body, &em) test.Nil(t, err) test.Equal(t, "MISSING_ARG_TOPIC", em.Message) topicName := "sampletopicA" + strconv.Itoa(int(time.Now().Unix())) + "$" url = fmt.Sprintf("http://%s/topic/create?topic=%s", nsqlookupd1.RealHTTPAddr(), topicName) req, _ = http.NewRequest("POST", url, nil) resp, err = client.Do(req) test.Nil(t, err) test.Equal(t, 400, resp.StatusCode) test.Equal(t, "Bad Request", http.StatusText(resp.StatusCode)) body, _ = io.ReadAll(resp.Body) resp.Body.Close() t.Logf("%s", body) err = json.Unmarshal(body, &em) test.Nil(t, err) test.Equal(t, "INVALID_ARG_TOPIC", em.Message) topicName = "sampletopicA" + strconv.Itoa(int(time.Now().Unix())) url = fmt.Sprintf("http://%s/topic/create?topic=%s", nsqlookupd1.RealHTTPAddr(), topicName) req, _ = http.NewRequest("POST", url, nil) resp, err = client.Do(req) test.Nil(t, err) test.Equal(t, 200, resp.StatusCode) body, _ = io.ReadAll(resp.Body) resp.Body.Close() t.Logf("%s", body) test.Equal(t, []byte(""), body) } func TestDeleteTopic(t *testing.T) { dataPath, nsqds, nsqlookupd1 := bootstrapNSQCluster(t) defer os.RemoveAll(dataPath) defer nsqds[0].Exit() defer nsqlookupd1.Exit() em := ErrMessage{} client := http.Client{} url := fmt.Sprintf("http://%s/topic/delete", nsqlookupd1.RealHTTPAddr()) req, _ := http.NewRequest("POST", url, nil) resp, err := client.Do(req) test.Nil(t, err) test.Equal(t, 400, resp.StatusCode) test.Equal(t, "Bad Request", http.StatusText(resp.StatusCode)) body, _ := io.ReadAll(resp.Body) resp.Body.Close() t.Logf("%s", body) err = json.Unmarshal(body, &em) test.Nil(t, err) test.Equal(t, "MISSING_ARG_TOPIC", em.Message) topicName := "sampletopicA" + strconv.Itoa(int(time.Now().Unix())) makeTopic(nsqlookupd1, topicName) url = fmt.Sprintf("http://%s/topic/delete?topic=%s", nsqlookupd1.RealHTTPAddr(), topicName) req, _ = http.NewRequest("POST", url, nil) resp, err = client.Do(req) test.Nil(t, err) test.Equal(t, 200, resp.StatusCode) body, _ = io.ReadAll(resp.Body) resp.Body.Close() t.Logf("%s", body) test.Equal(t, []byte(""), body) topicName = "sampletopicB" + strconv.Itoa(int(time.Now().Unix())) channelName := "foobar" + strconv.Itoa(int(time.Now().Unix())) makeChannel(nsqlookupd1, topicName, channelName) url = fmt.Sprintf("http://%s/topic/delete?topic=%s", nsqlookupd1.RealHTTPAddr(), topicName) req, _ = http.NewRequest("POST", url, nil) resp, err = client.Do(req) test.Nil(t, err) test.Equal(t, 200, resp.StatusCode) body, _ = io.ReadAll(resp.Body) resp.Body.Close() t.Logf("%s", body) test.Equal(t, []byte(""), body) } func TestGetChannels(t *testing.T) { dataPath, nsqds, nsqlookupd1 := bootstrapNSQCluster(t) defer os.RemoveAll(dataPath) defer nsqds[0].Exit() defer nsqlookupd1.Exit() client := http.Client{} url := fmt.Sprintf("http://%s/channels", nsqlookupd1.RealHTTPAddr()) em := ErrMessage{} req, _ := http.NewRequest("GET", url, nil) req.Header.Add("Accept", "application/vnd.nsq; version=1.0") resp, err := client.Do(req) test.Nil(t, err) test.Equal(t, 400, resp.StatusCode) test.Equal(t, "Bad Request", http.StatusText(resp.StatusCode)) body, _ := io.ReadAll(resp.Body) resp.Body.Close() t.Logf("%s", body) err = json.Unmarshal(body, &em) test.Nil(t, err) test.Equal(t, "MISSING_ARG_TOPIC", em.Message) ch := ChannelsDoc{} topicName := "sampletopicA" + strconv.Itoa(int(time.Now().Unix())) makeTopic(nsqlookupd1, topicName) url = fmt.Sprintf("http://%s/channels?topic=%s", nsqlookupd1.RealHTTPAddr(), topicName) req, _ = http.NewRequest("GET", url, nil) req.Header.Add("Accept", "application/vnd.nsq; version=1.0") resp, err = client.Do(req) test.Nil(t, err) test.Equal(t, 200, resp.StatusCode) body, _ = io.ReadAll(resp.Body) resp.Body.Close() t.Logf("%s", body) err = json.Unmarshal(body, &ch) test.Nil(t, err) test.Equal(t, 0, len(ch.Channels)) topicName = "sampletopicB" + strconv.Itoa(int(time.Now().Unix())) channelName := "foobar" + strconv.Itoa(int(time.Now().Unix())) makeChannel(nsqlookupd1, topicName, channelName) url = fmt.Sprintf("http://%s/channels?topic=%s", nsqlookupd1.RealHTTPAddr(), topicName) req, _ = http.NewRequest("GET", url, nil) req.Header.Add("Accept", "application/vnd.nsq; version=1.0") resp, err = client.Do(req) test.Nil(t, err) test.Equal(t, 200, resp.StatusCode) body, _ = io.ReadAll(resp.Body) resp.Body.Close() t.Logf("%s", body) err = json.Unmarshal(body, &ch) test.Nil(t, err) test.Equal(t, 1, len(ch.Channels)) test.Equal(t, channelName, ch.Channels[0]) } func TestCreateChannel(t *testing.T) { dataPath, nsqds, nsqlookupd1 := bootstrapNSQCluster(t) defer os.RemoveAll(dataPath) defer nsqds[0].Exit() defer nsqlookupd1.Exit() em := ErrMessage{} client := http.Client{} url := fmt.Sprintf("http://%s/channel/create", nsqlookupd1.RealHTTPAddr()) req, _ := http.NewRequest("POST", url, nil) resp, err := client.Do(req) test.Nil(t, err) test.Equal(t, 400, resp.StatusCode) test.Equal(t, "Bad Request", http.StatusText(resp.StatusCode)) body, _ := io.ReadAll(resp.Body) resp.Body.Close() t.Logf("%s", body) err = json.Unmarshal(body, &em) test.Nil(t, err) test.Equal(t, "MISSING_ARG_TOPIC", em.Message) topicName := "sampletopicB" + strconv.Itoa(int(time.Now().Unix())) + "$" url = fmt.Sprintf("http://%s/channel/create?topic=%s", nsqlookupd1.RealHTTPAddr(), topicName) req, _ = http.NewRequest("POST", url, nil) resp, err = client.Do(req) test.Nil(t, err) test.Equal(t, 400, resp.StatusCode) test.Equal(t, "Bad Request", http.StatusText(resp.StatusCode)) body, _ = io.ReadAll(resp.Body) resp.Body.Close() t.Logf("%s", body) err = json.Unmarshal(body, &em) test.Nil(t, err) test.Equal(t, "INVALID_ARG_TOPIC", em.Message) topicName = "sampletopicB" + strconv.Itoa(int(time.Now().Unix())) url = fmt.Sprintf("http://%s/channel/create?topic=%s", nsqlookupd1.RealHTTPAddr(), topicName) req, _ = http.NewRequest("POST", url, nil) resp, err = client.Do(req) test.Nil(t, err) test.Equal(t, 400, resp.StatusCode) test.Equal(t, "Bad Request", http.StatusText(resp.StatusCode)) body, _ = io.ReadAll(resp.Body) resp.Body.Close() t.Logf("%s", body) err = json.Unmarshal(body, &em) test.Nil(t, err) test.Equal(t, "MISSING_ARG_CHANNEL", em.Message) channelName := "foobar" + strconv.Itoa(int(time.Now().Unix())) + "$" url = fmt.Sprintf("http://%s/channel/create?topic=%s&channel=%s", nsqlookupd1.RealHTTPAddr(), topicName, channelName) req, _ = http.NewRequest("POST", url, nil) resp, err = client.Do(req) test.Nil(t, err) test.Equal(t, 400, resp.StatusCode) test.Equal(t, "Bad Request", http.StatusText(resp.StatusCode)) body, _ = io.ReadAll(resp.Body) resp.Body.Close() t.Logf("%s", body) err = json.Unmarshal(body, &em) test.Nil(t, err) test.Equal(t, "INVALID_ARG_CHANNEL", em.Message) channelName = "foobar" + strconv.Itoa(int(time.Now().Unix())) url = fmt.Sprintf("http://%s/channel/create?topic=%s&channel=%s", nsqlookupd1.RealHTTPAddr(), topicName, channelName) req, _ = http.NewRequest("POST", url, nil) resp, err = client.Do(req) test.Nil(t, err) test.Equal(t, 200, resp.StatusCode) body, _ = io.ReadAll(resp.Body) resp.Body.Close() t.Logf("%s", body) test.Equal(t, []byte(""), body) } func TestDeleteChannel(t *testing.T) { dataPath, nsqds, nsqlookupd1 := bootstrapNSQCluster(t) defer os.RemoveAll(dataPath) defer nsqds[0].Exit() defer nsqlookupd1.Exit() em := ErrMessage{} client := http.Client{} url := fmt.Sprintf("http://%s/channel/delete", nsqlookupd1.RealHTTPAddr()) req, _ := http.NewRequest("POST", url, nil) resp, err := client.Do(req) test.Nil(t, err) test.Equal(t, 400, resp.StatusCode) test.Equal(t, "Bad Request", http.StatusText(resp.StatusCode)) body, _ := io.ReadAll(resp.Body) resp.Body.Close() t.Logf("%s", body) err = json.Unmarshal(body, &em) test.Nil(t, err) test.Equal(t, "MISSING_ARG_TOPIC", em.Message) topicName := "sampletopicB" + strconv.Itoa(int(time.Now().Unix())) + "$" url = fmt.Sprintf("http://%s/channel/delete?topic=%s", nsqlookupd1.RealHTTPAddr(), topicName) req, _ = http.NewRequest("POST", url, nil) resp, err = client.Do(req) test.Nil(t, err) test.Equal(t, 400, resp.StatusCode) test.Equal(t, "Bad Request", http.StatusText(resp.StatusCode)) body, _ = io.ReadAll(resp.Body) resp.Body.Close() t.Logf("%s", body) err = json.Unmarshal(body, &em) test.Nil(t, err) test.Equal(t, "INVALID_ARG_TOPIC", em.Message) topicName = "sampletopicB" + strconv.Itoa(int(time.Now().Unix())) url = fmt.Sprintf("http://%s/channel/delete?topic=%s", nsqlookupd1.RealHTTPAddr(), topicName) req, _ = http.NewRequest("POST", url, nil) resp, err = client.Do(req) test.Nil(t, err) test.Equal(t, 400, resp.StatusCode) test.Equal(t, "Bad Request", http.StatusText(resp.StatusCode)) body, _ = io.ReadAll(resp.Body) resp.Body.Close() t.Logf("%s", body) err = json.Unmarshal(body, &em) test.Nil(t, err) test.Equal(t, "MISSING_ARG_CHANNEL", em.Message) channelName := "foobar" + strconv.Itoa(int(time.Now().Unix())) + "$" url = fmt.Sprintf("http://%s/channel/delete?topic=%s&channel=%s", nsqlookupd1.RealHTTPAddr(), topicName, channelName) req, _ = http.NewRequest("POST", url, nil) resp, err = client.Do(req) test.Nil(t, err) test.Equal(t, 400, resp.StatusCode) test.Equal(t, "Bad Request", http.StatusText(resp.StatusCode)) body, _ = io.ReadAll(resp.Body) resp.Body.Close() t.Logf("%s", body) err = json.Unmarshal(body, &em) test.Nil(t, err) test.Equal(t, "INVALID_ARG_CHANNEL", em.Message) channelName = "foobar" + strconv.Itoa(int(time.Now().Unix())) url = fmt.Sprintf("http://%s/channel/delete?topic=%s&channel=%s", nsqlookupd1.RealHTTPAddr(), topicName, channelName) req, _ = http.NewRequest("POST", url, nil) resp, err = client.Do(req) test.Nil(t, err) test.Equal(t, 404, resp.StatusCode) test.Equal(t, "Not Found", http.StatusText(resp.StatusCode)) body, _ = io.ReadAll(resp.Body) resp.Body.Close() t.Logf("%s", body) err = json.Unmarshal(body, &em) test.Nil(t, err) test.Equal(t, "CHANNEL_NOT_FOUND", em.Message) makeChannel(nsqlookupd1, topicName, channelName) req, _ = http.NewRequest("POST", url, nil) resp, err = client.Do(req) test.Nil(t, err) test.Equal(t, 200, resp.StatusCode) body, _ = io.ReadAll(resp.Body) resp.Body.Close() t.Logf("%s", body) test.Equal(t, []byte(""), body) } ================================================ FILE: nsqlookupd/logger.go ================================================ package nsqlookupd import ( "github.com/nsqio/nsq/internal/lg" ) type Logger lg.Logger const ( LOG_DEBUG = lg.DEBUG LOG_INFO = lg.INFO LOG_WARN = lg.WARN LOG_ERROR = lg.ERROR LOG_FATAL = lg.FATAL ) func (n *NSQLookupd) logf(level lg.LogLevel, f string, args ...interface{}) { lg.Logf(n.opts.Logger, n.opts.LogLevel, level, f, args...) } ================================================ FILE: nsqlookupd/lookup_protocol_v1.go ================================================ package nsqlookupd import ( "bufio" "encoding/binary" "encoding/json" "fmt" "io" "log" "net" "os" "strings" "sync/atomic" "time" "github.com/nsqio/nsq/internal/protocol" "github.com/nsqio/nsq/internal/version" ) type LookupProtocolV1 struct { nsqlookupd *NSQLookupd } func (p *LookupProtocolV1) NewClient(conn net.Conn) protocol.Client { return NewClientV1(conn) } func (p *LookupProtocolV1) IOLoop(c protocol.Client) error { var err error var line string client := c.(*ClientV1) reader := bufio.NewReader(client) for { line, err = reader.ReadString('\n') if err != nil { break } line = strings.TrimSpace(line) params := strings.Split(line, " ") var response []byte response, err = p.Exec(client, reader, params) if err != nil { ctx := "" if parentErr := err.(protocol.ChildErr).Parent(); parentErr != nil { ctx = " - " + parentErr.Error() } p.nsqlookupd.logf(LOG_ERROR, "[%s] - %s%s", client, err, ctx) _, sendErr := protocol.SendResponse(client, []byte(err.Error())) if sendErr != nil { p.nsqlookupd.logf(LOG_ERROR, "[%s] - %s%s", client, sendErr, ctx) break } // errors of type FatalClientErr should forceably close the connection if _, ok := err.(*protocol.FatalClientErr); ok { break } continue } if response != nil { _, err = protocol.SendResponse(client, response) if err != nil { break } } } p.nsqlookupd.logf(LOG_INFO, "PROTOCOL(V1): [%s] exiting ioloop", client) if client.peerInfo != nil { registrations := p.nsqlookupd.DB.LookupRegistrations(client.peerInfo.id) for _, r := range registrations { if removed, _ := p.nsqlookupd.DB.RemoveProducer(r, client.peerInfo.id); removed { p.nsqlookupd.logf(LOG_INFO, "DB: client(%s) UNREGISTER category:%s key:%s subkey:%s", client, r.Category, r.Key, r.SubKey) } } } return err } func (p *LookupProtocolV1) Exec(client *ClientV1, reader *bufio.Reader, params []string) ([]byte, error) { switch params[0] { case "PING": return p.PING(client, params) case "IDENTIFY": return p.IDENTIFY(client, reader, params[1:]) case "REGISTER": return p.REGISTER(client, reader, params[1:]) case "UNREGISTER": return p.UNREGISTER(client, reader, params[1:]) } return nil, protocol.NewFatalClientErr(nil, "E_INVALID", fmt.Sprintf("invalid command %s", params[0])) } func getTopicChan(command string, params []string) (string, string, error) { if len(params) == 0 { return "", "", protocol.NewFatalClientErr(nil, "E_INVALID", fmt.Sprintf("%s insufficient number of params", command)) } topicName := params[0] var channelName string if len(params) >= 2 { channelName = params[1] } if !protocol.IsValidTopicName(topicName) { return "", "", protocol.NewFatalClientErr(nil, "E_BAD_TOPIC", fmt.Sprintf("%s topic name '%s' is not valid", command, topicName)) } if channelName != "" && !protocol.IsValidChannelName(channelName) { return "", "", protocol.NewFatalClientErr(nil, "E_BAD_CHANNEL", fmt.Sprintf("%s channel name '%s' is not valid", command, channelName)) } return topicName, channelName, nil } func (p *LookupProtocolV1) REGISTER(client *ClientV1, reader *bufio.Reader, params []string) ([]byte, error) { if client.peerInfo == nil { return nil, protocol.NewFatalClientErr(nil, "E_INVALID", "client must IDENTIFY") } topic, channel, err := getTopicChan("REGISTER", params) if err != nil { return nil, err } if channel != "" { key := Registration{"channel", topic, channel} if p.nsqlookupd.DB.AddProducer(key, &Producer{peerInfo: client.peerInfo}) { p.nsqlookupd.logf(LOG_INFO, "DB: client(%s) REGISTER category:%s key:%s subkey:%s", client, "channel", topic, channel) } } key := Registration{"topic", topic, ""} if p.nsqlookupd.DB.AddProducer(key, &Producer{peerInfo: client.peerInfo}) { p.nsqlookupd.logf(LOG_INFO, "DB: client(%s) REGISTER category:%s key:%s subkey:%s", client, "topic", topic, "") } return []byte("OK"), nil } func (p *LookupProtocolV1) UNREGISTER(client *ClientV1, reader *bufio.Reader, params []string) ([]byte, error) { if client.peerInfo == nil { return nil, protocol.NewFatalClientErr(nil, "E_INVALID", "client must IDENTIFY") } topic, channel, err := getTopicChan("UNREGISTER", params) if err != nil { return nil, err } if channel != "" { key := Registration{"channel", topic, channel} removed, left := p.nsqlookupd.DB.RemoveProducer(key, client.peerInfo.id) if removed { p.nsqlookupd.logf(LOG_INFO, "DB: client(%s) UNREGISTER category:%s key:%s subkey:%s", client, "channel", topic, channel) } // for ephemeral channels, remove the channel as well if it has no producers if left == 0 && strings.HasSuffix(channel, "#ephemeral") { p.nsqlookupd.DB.RemoveRegistration(key) } } else { // no channel was specified so this is a topic unregistration // remove all of the channel registrations... // normally this shouldn't happen which is why we print a warning message // if anything is actually removed registrations := p.nsqlookupd.DB.FindRegistrations("channel", topic, "*") for _, r := range registrations { removed, _ := p.nsqlookupd.DB.RemoveProducer(r, client.peerInfo.id) if removed { p.nsqlookupd.logf(LOG_WARN, "client(%s) unexpected UNREGISTER category:%s key:%s subkey:%s", client, "channel", topic, r.SubKey) } } key := Registration{"topic", topic, ""} removed, left := p.nsqlookupd.DB.RemoveProducer(key, client.peerInfo.id) if removed { p.nsqlookupd.logf(LOG_INFO, "DB: client(%s) UNREGISTER category:%s key:%s subkey:%s", client, "topic", topic, "") } if left == 0 && strings.HasSuffix(topic, "#ephemeral") { p.nsqlookupd.DB.RemoveRegistration(key) } } return []byte("OK"), nil } func (p *LookupProtocolV1) IDENTIFY(client *ClientV1, reader *bufio.Reader, params []string) ([]byte, error) { var err error if client.peerInfo != nil { return nil, protocol.NewFatalClientErr(err, "E_INVALID", "cannot IDENTIFY again") } var bodyLen int32 err = binary.Read(reader, binary.BigEndian, &bodyLen) if err != nil { return nil, protocol.NewFatalClientErr(err, "E_BAD_BODY", "IDENTIFY failed to read body size") } body := make([]byte, bodyLen) _, err = io.ReadFull(reader, body) if err != nil { return nil, protocol.NewFatalClientErr(err, "E_BAD_BODY", "IDENTIFY failed to read body") } // body is a json structure with producer information peerInfo := PeerInfo{id: client.RemoteAddr().String()} err = json.Unmarshal(body, &peerInfo) if err != nil { return nil, protocol.NewFatalClientErr(err, "E_BAD_BODY", "IDENTIFY failed to decode JSON body") } peerInfo.RemoteAddress = client.RemoteAddr().String() // require all fields if peerInfo.BroadcastAddress == "" || peerInfo.TCPPort == 0 || peerInfo.HTTPPort == 0 || peerInfo.Version == "" { return nil, protocol.NewFatalClientErr(nil, "E_BAD_BODY", "IDENTIFY missing fields") } atomic.StoreInt64(&peerInfo.lastUpdate, time.Now().UnixNano()) p.nsqlookupd.logf(LOG_INFO, "CLIENT(%s): IDENTIFY Address:%s TCP:%d HTTP:%d Version:%s", client, peerInfo.BroadcastAddress, peerInfo.TCPPort, peerInfo.HTTPPort, peerInfo.Version) client.peerInfo = &peerInfo if p.nsqlookupd.DB.AddProducer(Registration{"client", "", ""}, &Producer{peerInfo: client.peerInfo}) { p.nsqlookupd.logf(LOG_INFO, "DB: client(%s) REGISTER category:%s key:%s subkey:%s", client, "client", "", "") } // build a response data := make(map[string]interface{}) data["tcp_port"] = p.nsqlookupd.RealTCPAddr().Port data["http_port"] = p.nsqlookupd.RealHTTPAddr().Port data["version"] = version.Binary hostname, err := os.Hostname() if err != nil { log.Fatalf("ERROR: unable to get hostname %s", err) } data["broadcast_address"] = p.nsqlookupd.opts.BroadcastAddress data["hostname"] = hostname response, err := json.Marshal(data) if err != nil { p.nsqlookupd.logf(LOG_ERROR, "marshaling %v", data) return []byte("OK"), nil } return response, nil } func (p *LookupProtocolV1) PING(client *ClientV1, params []string) ([]byte, error) { if client.peerInfo != nil { // we could get a PING before other commands on the same client connection cur := time.Unix(0, atomic.LoadInt64(&client.peerInfo.lastUpdate)) now := time.Now() p.nsqlookupd.logf(LOG_INFO, "CLIENT(%s): pinged (last ping %s)", client.peerInfo.id, now.Sub(cur)) atomic.StoreInt64(&client.peerInfo.lastUpdate, now.UnixNano()) } return []byte("OK"), nil } ================================================ FILE: nsqlookupd/lookup_protocol_v1_test.go ================================================ package nsqlookupd import ( "errors" "testing" "time" "github.com/nsqio/nsq/internal/protocol" "github.com/nsqio/nsq/internal/test" ) func TestIOLoopReturnsClientErrWhenSendFails(t *testing.T) { fakeConn := test.NewFakeNetConn() fakeConn.WriteFunc = func(b []byte) (int, error) { return 0, errors.New("write error") } testIOLoopReturnsClientErr(t, fakeConn) } func TestIOLoopReturnsClientErrWhenSendSucceeds(t *testing.T) { fakeConn := test.NewFakeNetConn() fakeConn.WriteFunc = func(b []byte) (int, error) { return len(b), nil } testIOLoopReturnsClientErr(t, fakeConn) } func testIOLoopReturnsClientErr(t *testing.T, fakeConn test.FakeNetConn) { fakeConn.ReadFunc = func(b []byte) (int, error) { return copy(b, []byte("INVALID_COMMAND\n")), nil } opts := NewOptions() opts.Logger = test.NewTestLogger(t) opts.LogLevel = LOG_DEBUG opts.TCPAddress = "127.0.0.1:0" opts.HTTPAddress = "127.0.0.1:0" nsqlookupd, err := New(opts) test.Nil(t, err) prot := &LookupProtocolV1{nsqlookupd: nsqlookupd} nsqlookupd.tcpServer = &tcpServer{nsqlookupd: prot.nsqlookupd} errChan := make(chan error) testIOLoop := func() { client := prot.NewClient(fakeConn) errChan <- prot.IOLoop(client) defer prot.nsqlookupd.Exit() } go testIOLoop() var timeout bool select { case err = <-errChan: case <-time.After(2 * time.Second): timeout = true } test.Equal(t, false, timeout) test.NotNil(t, err) test.Equal(t, "E_INVALID invalid command INVALID_COMMAND", err.Error()) test.NotNil(t, err.(*protocol.FatalClientErr)) } ================================================ FILE: nsqlookupd/nsqlookupd.go ================================================ package nsqlookupd import ( "fmt" "log" "net" "os" "sync" "github.com/nsqio/nsq/internal/http_api" "github.com/nsqio/nsq/internal/protocol" "github.com/nsqio/nsq/internal/util" "github.com/nsqio/nsq/internal/version" ) type NSQLookupd struct { sync.RWMutex opts *Options tcpListener net.Listener httpListener net.Listener tcpServer *tcpServer waitGroup util.WaitGroupWrapper DB *RegistrationDB } func New(opts *Options) (*NSQLookupd, error) { var err error if opts.Logger == nil { opts.Logger = log.New(os.Stderr, opts.LogPrefix, log.Ldate|log.Ltime|log.Lmicroseconds) } l := &NSQLookupd{ opts: opts, DB: NewRegistrationDB(), } l.logf(LOG_INFO, version.String("nsqlookupd")) l.tcpServer = &tcpServer{nsqlookupd: l} l.tcpListener, err = net.Listen("tcp", opts.TCPAddress) if err != nil { return nil, fmt.Errorf("listen (%s) failed - %s", opts.TCPAddress, err) } l.httpListener, err = net.Listen("tcp", opts.HTTPAddress) if err != nil { return nil, fmt.Errorf("listen (%s) failed - %s", opts.HTTPAddress, err) } return l, nil } // Main starts an instance of nsqlookupd and returns an // error if there was a problem starting up. func (l *NSQLookupd) Main() error { exitCh := make(chan error) var once sync.Once exitFunc := func(err error) { once.Do(func() { if err != nil { l.logf(LOG_FATAL, "%s", err) } exitCh <- err }) } l.waitGroup.Wrap(func() { exitFunc(protocol.TCPServer(l.tcpListener, l.tcpServer, l.logf)) }) httpServer := newHTTPServer(l) l.waitGroup.Wrap(func() { exitFunc(http_api.Serve(l.httpListener, httpServer, "HTTP", l.logf)) }) err := <-exitCh return err } func (l *NSQLookupd) RealTCPAddr() *net.TCPAddr { return l.tcpListener.Addr().(*net.TCPAddr) } func (l *NSQLookupd) RealHTTPAddr() *net.TCPAddr { return l.httpListener.Addr().(*net.TCPAddr) } func (l *NSQLookupd) Exit() { if l.tcpListener != nil { l.tcpListener.Close() } if l.tcpServer != nil { l.tcpServer.Close() } if l.httpListener != nil { l.httpListener.Close() } l.waitGroup.Wait() } ================================================ FILE: nsqlookupd/nsqlookupd_test.go ================================================ package nsqlookupd import ( "fmt" "net" "testing" "time" "github.com/nsqio/go-nsq" "github.com/nsqio/nsq/internal/clusterinfo" "github.com/nsqio/nsq/internal/http_api" "github.com/nsqio/nsq/internal/test" ) const ( ConnectTimeout = 2 * time.Second RequestTimeout = 5 * time.Second TCPPort = 5000 HTTPPort = 5555 HostAddr = "ip.address" NSQDVersion = "fake-version" ) type ProducersDoc struct { Producers []interface{} `json:"producers"` } type TopicsDoc struct { Topics []interface{} `json:"topics"` } type LookupDoc struct { Channels []interface{} `json:"channels"` Producers []*PeerInfo `json:"producers"` } func mustStartLookupd(opts *Options) (*net.TCPAddr, *net.TCPAddr, *NSQLookupd) { opts.TCPAddress = "127.0.0.1:0" opts.HTTPAddress = "127.0.0.1:0" nsqlookupd, err := New(opts) if err != nil { panic(err) } go func() { err := nsqlookupd.Main() if err != nil { panic(err) } }() return nsqlookupd.RealTCPAddr(), nsqlookupd.RealHTTPAddr(), nsqlookupd } func mustConnectLookupd(t *testing.T, tcpAddr *net.TCPAddr) net.Conn { conn, err := net.DialTimeout("tcp", tcpAddr.String(), time.Second) if err != nil { t.Fatal("failed to connect to lookupd") } conn.Write(nsq.MagicV1) return conn } func identify(t *testing.T, conn net.Conn) { ci := make(map[string]interface{}) ci["tcp_port"] = TCPPort ci["http_port"] = HTTPPort ci["broadcast_address"] = HostAddr ci["hostname"] = HostAddr ci["version"] = NSQDVersion cmd, _ := nsq.Identify(ci) _, err := cmd.WriteTo(conn) test.Nil(t, err) _, err = nsq.ReadResponse(conn) test.Nil(t, err) } func TestBasicLookupd(t *testing.T) { opts := NewOptions() opts.Logger = test.NewTestLogger(t) tcpAddr, httpAddr, nsqlookupd := mustStartLookupd(opts) defer nsqlookupd.Exit() topics := nsqlookupd.DB.FindRegistrations("topic", "*", "*") test.Equal(t, 0, len(topics)) topicName := "connectmsg" conn := mustConnectLookupd(t, tcpAddr) identify(t, conn) nsq.Register(topicName, "channel1").WriteTo(conn) v, err := nsq.ReadResponse(conn) test.Nil(t, err) test.Equal(t, []byte("OK"), v) pr := ProducersDoc{} endpoint := fmt.Sprintf("http://%s/nodes", httpAddr) err = http_api.NewClient(nil, ConnectTimeout, RequestTimeout).GETV1(endpoint, &pr) test.Nil(t, err) t.Logf("got %v", pr) test.Equal(t, 1, len(pr.Producers)) topics = nsqlookupd.DB.FindRegistrations("topic", topicName, "") test.Equal(t, 1, len(topics)) producers := nsqlookupd.DB.FindProducers("topic", topicName, "") test.Equal(t, 1, len(producers)) producer := producers[0] test.Equal(t, HostAddr, producer.peerInfo.BroadcastAddress) test.Equal(t, HostAddr, producer.peerInfo.Hostname) test.Equal(t, TCPPort, producer.peerInfo.TCPPort) test.Equal(t, HTTPPort, producer.peerInfo.HTTPPort) tr := TopicsDoc{} endpoint = fmt.Sprintf("http://%s/topics", httpAddr) err = http_api.NewClient(nil, ConnectTimeout, RequestTimeout).GETV1(endpoint, &tr) test.Nil(t, err) t.Logf("got %v", tr) test.Equal(t, 1, len(tr.Topics)) lr := LookupDoc{} endpoint = fmt.Sprintf("http://%s/lookup?topic=%s", httpAddr, topicName) err = http_api.NewClient(nil, ConnectTimeout, RequestTimeout).GETV1(endpoint, &lr) test.Nil(t, err) t.Logf("got %v", lr) test.Equal(t, 1, len(lr.Channels)) test.Equal(t, 1, len(lr.Producers)) for _, p := range lr.Producers { test.Equal(t, TCPPort, p.TCPPort) test.Equal(t, HTTPPort, p.HTTPPort) test.Equal(t, HostAddr, p.BroadcastAddress) test.Equal(t, NSQDVersion, p.Version) } conn.Close() time.Sleep(10 * time.Millisecond) // now there should be no producers, but still topic/channel entries err = http_api.NewClient(nil, ConnectTimeout, RequestTimeout).GETV1(endpoint, &lr) test.Nil(t, err) test.Equal(t, 1, len(lr.Channels)) test.Equal(t, 0, len(lr.Producers)) } func TestChannelUnregister(t *testing.T) { opts := NewOptions() opts.Logger = test.NewTestLogger(t) tcpAddr, httpAddr, nsqlookupd := mustStartLookupd(opts) defer nsqlookupd.Exit() topics := nsqlookupd.DB.FindRegistrations("topic", "*", "*") test.Equal(t, 0, len(topics)) topicName := "channel_unregister" conn := mustConnectLookupd(t, tcpAddr) defer conn.Close() identify(t, conn) nsq.Register(topicName, "ch1").WriteTo(conn) v, err := nsq.ReadResponse(conn) test.Nil(t, err) test.Equal(t, []byte("OK"), v) topics = nsqlookupd.DB.FindRegistrations("topic", topicName, "") test.Equal(t, 1, len(topics)) channels := nsqlookupd.DB.FindRegistrations("channel", topicName, "*") test.Equal(t, 1, len(channels)) nsq.UnRegister(topicName, "ch1").WriteTo(conn) v, err = nsq.ReadResponse(conn) test.Nil(t, err) test.Equal(t, []byte("OK"), v) topics = nsqlookupd.DB.FindRegistrations("topic", topicName, "") test.Equal(t, 1, len(topics)) // we should still have mention of the topic even though there is no producer // (ie. we haven't *deleted* the channel, just unregistered as a producer) channels = nsqlookupd.DB.FindRegistrations("channel", topicName, "*") test.Equal(t, 1, len(channels)) pr := ProducersDoc{} endpoint := fmt.Sprintf("http://%s/lookup?topic=%s", httpAddr, topicName) err = http_api.NewClient(nil, ConnectTimeout, RequestTimeout).GETV1(endpoint, &pr) test.Nil(t, err) t.Logf("got %v", pr) test.Equal(t, 1, len(pr.Producers)) } func TestTombstoneRecover(t *testing.T) { opts := NewOptions() opts.Logger = test.NewTestLogger(t) opts.TombstoneLifetime = 50 * time.Millisecond tcpAddr, httpAddr, nsqlookupd := mustStartLookupd(opts) defer nsqlookupd.Exit() topicName := "tombstone_recover" topicName2 := topicName + "2" conn := mustConnectLookupd(t, tcpAddr) defer conn.Close() identify(t, conn) nsq.Register(topicName, "channel1").WriteTo(conn) _, err := nsq.ReadResponse(conn) test.Nil(t, err) nsq.Register(topicName2, "channel2").WriteTo(conn) _, err = nsq.ReadResponse(conn) test.Nil(t, err) endpoint := fmt.Sprintf("http://%s/topic/tombstone?topic=%s&node=%s:%d", httpAddr, topicName, HostAddr, HTTPPort) err = http_api.NewClient(nil, ConnectTimeout, RequestTimeout).POSTV1(endpoint, nil, nil) test.Nil(t, err) pr := ProducersDoc{} endpoint = fmt.Sprintf("http://%s/lookup?topic=%s", httpAddr, topicName) err = http_api.NewClient(nil, ConnectTimeout, RequestTimeout).GETV1(endpoint, &pr) test.Nil(t, err) test.Equal(t, 0, len(pr.Producers)) endpoint = fmt.Sprintf("http://%s/lookup?topic=%s", httpAddr, topicName2) err = http_api.NewClient(nil, ConnectTimeout, RequestTimeout).GETV1(endpoint, &pr) test.Nil(t, err) test.Equal(t, 1, len(pr.Producers)) time.Sleep(75 * time.Millisecond) endpoint = fmt.Sprintf("http://%s/lookup?topic=%s", httpAddr, topicName) err = http_api.NewClient(nil, ConnectTimeout, RequestTimeout).GETV1(endpoint, &pr) test.Nil(t, err) test.Equal(t, 1, len(pr.Producers)) } func TestTombstoneUnregister(t *testing.T) { opts := NewOptions() opts.Logger = test.NewTestLogger(t) opts.TombstoneLifetime = 50 * time.Millisecond tcpAddr, httpAddr, nsqlookupd := mustStartLookupd(opts) defer nsqlookupd.Exit() topicName := "tombstone_unregister" conn := mustConnectLookupd(t, tcpAddr) defer conn.Close() identify(t, conn) nsq.Register(topicName, "channel1").WriteTo(conn) _, err := nsq.ReadResponse(conn) test.Nil(t, err) endpoint := fmt.Sprintf("http://%s/topic/tombstone?topic=%s&node=%s:%d", httpAddr, topicName, HostAddr, HTTPPort) err = http_api.NewClient(nil, ConnectTimeout, RequestTimeout).POSTV1(endpoint, nil, nil) test.Nil(t, err) pr := ProducersDoc{} endpoint = fmt.Sprintf("http://%s/lookup?topic=%s", httpAddr, topicName) err = http_api.NewClient(nil, ConnectTimeout, RequestTimeout).GETV1(endpoint, &pr) test.Nil(t, err) test.Equal(t, 0, len(pr.Producers)) nsq.UnRegister(topicName, "").WriteTo(conn) _, err = nsq.ReadResponse(conn) test.Nil(t, err) time.Sleep(55 * time.Millisecond) endpoint = fmt.Sprintf("http://%s/lookup?topic=%s", httpAddr, topicName) err = http_api.NewClient(nil, ConnectTimeout, RequestTimeout).GETV1(endpoint, &pr) test.Nil(t, err) test.Equal(t, 0, len(pr.Producers)) } func TestInactiveNodes(t *testing.T) { opts := NewOptions() opts.Logger = test.NewTestLogger(t) opts.InactiveProducerTimeout = 200 * time.Millisecond tcpAddr, httpAddr, nsqlookupd := mustStartLookupd(opts) defer nsqlookupd.Exit() lookupdHTTPAddrs := []string{httpAddr.String()} topicName := "inactive_nodes" conn := mustConnectLookupd(t, tcpAddr) defer conn.Close() identify(t, conn) nsq.Register(topicName, "channel1").WriteTo(conn) _, err := nsq.ReadResponse(conn) test.Nil(t, err) ci := clusterinfo.New(nil, http_api.NewClient(nil, ConnectTimeout, RequestTimeout)) producers, _ := ci.GetLookupdProducers(lookupdHTTPAddrs) test.Equal(t, 1, len(producers)) test.Equal(t, 1, len(producers[0].Topics)) test.Equal(t, topicName, producers[0].Topics[0].Topic) test.Equal(t, false, producers[0].Topics[0].Tombstoned) time.Sleep(250 * time.Millisecond) producers, _ = ci.GetLookupdProducers(lookupdHTTPAddrs) test.Equal(t, 0, len(producers)) } func TestTombstonedNodes(t *testing.T) { opts := NewOptions() opts.Logger = test.NewTestLogger(t) tcpAddr, httpAddr, nsqlookupd := mustStartLookupd(opts) defer nsqlookupd.Exit() lookupdHTTPAddrs := []string{httpAddr.String()} topicName := "inactive_nodes" conn := mustConnectLookupd(t, tcpAddr) defer conn.Close() identify(t, conn) nsq.Register(topicName, "channel1").WriteTo(conn) _, err := nsq.ReadResponse(conn) test.Nil(t, err) ci := clusterinfo.New(nil, http_api.NewClient(nil, ConnectTimeout, RequestTimeout)) producers, _ := ci.GetLookupdProducers(lookupdHTTPAddrs) test.Equal(t, 1, len(producers)) test.Equal(t, 1, len(producers[0].Topics)) test.Equal(t, topicName, producers[0].Topics[0].Topic) test.Equal(t, false, producers[0].Topics[0].Tombstoned) endpoint := fmt.Sprintf("http://%s/topic/tombstone?topic=%s&node=%s:%d", httpAddr, topicName, HostAddr, HTTPPort) err = http_api.NewClient(nil, ConnectTimeout, RequestTimeout).POSTV1(endpoint, nil, nil) test.Nil(t, err) producers, _ = ci.GetLookupdProducers(lookupdHTTPAddrs) test.Equal(t, 1, len(producers)) test.Equal(t, 1, len(producers[0].Topics)) test.Equal(t, topicName, producers[0].Topics[0].Topic) test.Equal(t, true, producers[0].Topics[0].Tombstoned) } ================================================ FILE: nsqlookupd/options.go ================================================ package nsqlookupd import ( "log" "os" "time" "github.com/nsqio/nsq/internal/lg" ) type Options struct { LogLevel lg.LogLevel `flag:"log-level"` LogPrefix string `flag:"log-prefix"` Logger Logger TCPAddress string `flag:"tcp-address"` HTTPAddress string `flag:"http-address"` BroadcastAddress string `flag:"broadcast-address"` InactiveProducerTimeout time.Duration `flag:"inactive-producer-timeout"` TombstoneLifetime time.Duration `flag:"tombstone-lifetime"` } func NewOptions() *Options { hostname, err := os.Hostname() if err != nil { log.Fatal(err) } return &Options{ LogPrefix: "[nsqlookupd] ", LogLevel: lg.INFO, TCPAddress: "0.0.0.0:4160", HTTPAddress: "0.0.0.0:4161", BroadcastAddress: hostname, InactiveProducerTimeout: 300 * time.Second, TombstoneLifetime: 45 * time.Second, } } ================================================ FILE: nsqlookupd/registration_db.go ================================================ package nsqlookupd import ( "fmt" "sync" "sync/atomic" "time" ) type RegistrationDB struct { sync.RWMutex registrationMap map[Registration]ProducerMap } type Registration struct { Category string Key string SubKey string } type Registrations []Registration type PeerInfo struct { lastUpdate int64 id string RemoteAddress string `json:"remote_address"` Hostname string `json:"hostname"` BroadcastAddress string `json:"broadcast_address"` TCPPort int `json:"tcp_port"` HTTPPort int `json:"http_port"` Version string `json:"version"` TopologyZone string `json:"topology_zone"` TopologyRegion string `json:"topology_region"` } type Producer struct { peerInfo *PeerInfo tombstoned bool tombstonedAt time.Time } type Producers []*Producer type ProducerMap map[string]*Producer func (p *Producer) String() string { return fmt.Sprintf("%s [%d, %d]", p.peerInfo.BroadcastAddress, p.peerInfo.TCPPort, p.peerInfo.HTTPPort) } func (p *Producer) Tombstone() { p.tombstoned = true p.tombstonedAt = time.Now() } func (p *Producer) IsTombstoned(lifetime time.Duration) bool { return p.tombstoned && time.Since(p.tombstonedAt) < lifetime } func NewRegistrationDB() *RegistrationDB { return &RegistrationDB{ registrationMap: make(map[Registration]ProducerMap), } } // add a registration key func (r *RegistrationDB) AddRegistration(k Registration) { r.Lock() defer r.Unlock() _, ok := r.registrationMap[k] if !ok { r.registrationMap[k] = make(map[string]*Producer) } } // add a producer to a registration func (r *RegistrationDB) AddProducer(k Registration, p *Producer) bool { r.Lock() defer r.Unlock() _, ok := r.registrationMap[k] if !ok { r.registrationMap[k] = make(map[string]*Producer) } producers := r.registrationMap[k] _, found := producers[p.peerInfo.id] if !found { producers[p.peerInfo.id] = p } return !found } // remove a producer from a registration func (r *RegistrationDB) RemoveProducer(k Registration, id string) (bool, int) { r.Lock() defer r.Unlock() producers, ok := r.registrationMap[k] if !ok { return false, 0 } removed := false if _, exists := producers[id]; exists { removed = true } // Note: this leaves keys in the DB even if they have empty lists delete(producers, id) return removed, len(producers) } // remove a Registration and all it's producers func (r *RegistrationDB) RemoveRegistration(k Registration) { r.Lock() defer r.Unlock() delete(r.registrationMap, k) } func (r *RegistrationDB) needFilter(key string, subkey string) bool { return key == "*" || subkey == "*" } func (r *RegistrationDB) FindRegistrations(category string, key string, subkey string) Registrations { r.RLock() defer r.RUnlock() if !r.needFilter(key, subkey) { k := Registration{category, key, subkey} if _, ok := r.registrationMap[k]; ok { return Registrations{k} } return Registrations{} } results := Registrations{} for k := range r.registrationMap { if !k.IsMatch(category, key, subkey) { continue } results = append(results, k) } return results } func (r *RegistrationDB) FindProducers(category string, key string, subkey string) Producers { r.RLock() defer r.RUnlock() if !r.needFilter(key, subkey) { k := Registration{category, key, subkey} return ProducerMap2Slice(r.registrationMap[k]) } results := make(map[string]struct{}) var retProducers Producers for k, producers := range r.registrationMap { if !k.IsMatch(category, key, subkey) { continue } for _, producer := range producers { _, found := results[producer.peerInfo.id] if !found { results[producer.peerInfo.id] = struct{}{} retProducers = append(retProducers, producer) } } } return retProducers } func (r *RegistrationDB) LookupRegistrations(id string) Registrations { r.RLock() defer r.RUnlock() results := Registrations{} for k, producers := range r.registrationMap { if _, exists := producers[id]; exists { results = append(results, k) } } return results } func (k Registration) IsMatch(category string, key string, subkey string) bool { if category != k.Category { return false } if key != "*" && k.Key != key { return false } if subkey != "*" && k.SubKey != subkey { return false } return true } func (rr Registrations) Filter(category string, key string, subkey string) Registrations { output := Registrations{} for _, k := range rr { if k.IsMatch(category, key, subkey) { output = append(output, k) } } return output } func (rr Registrations) Keys() []string { keys := make([]string, len(rr)) for i, k := range rr { keys[i] = k.Key } return keys } func (rr Registrations) SubKeys() []string { subkeys := make([]string, len(rr)) for i, k := range rr { subkeys[i] = k.SubKey } return subkeys } func (pp Producers) FilterByActive(inactivityTimeout time.Duration, tombstoneLifetime time.Duration) Producers { now := time.Now() results := Producers{} for _, p := range pp { cur := time.Unix(0, atomic.LoadInt64(&p.peerInfo.lastUpdate)) if now.Sub(cur) > inactivityTimeout || p.IsTombstoned(tombstoneLifetime) { continue } results = append(results, p) } return results } func (pp Producers) PeerInfo() []*PeerInfo { results := []*PeerInfo{} for _, p := range pp { results = append(results, p.peerInfo) } return results } func ProducerMap2Slice(pm ProducerMap) Producers { var producers Producers for _, producer := range pm { producers = append(producers, producer) } return producers } ================================================ FILE: nsqlookupd/registration_db_test.go ================================================ package nsqlookupd import ( "math/rand" "strconv" "testing" "time" "github.com/nsqio/nsq/internal/test" ) func TestRegistrationDB(t *testing.T) { sec30 := 30 * time.Second beginningOfTime := time.Unix(1348797047, 0) pi1 := &PeerInfo{beginningOfTime.UnixNano(), "1", "remote_addr:1", "host", "b_addr", 1, 2, "v1", "", ""} pi2 := &PeerInfo{beginningOfTime.UnixNano(), "2", "remote_addr:2", "host", "b_addr", 2, 3, "v1", "", ""} pi3 := &PeerInfo{beginningOfTime.UnixNano(), "3", "remote_addr:3", "host", "b_addr", 3, 4, "v1", "", ""} p1 := &Producer{pi1, false, beginningOfTime} p2 := &Producer{pi2, false, beginningOfTime} p3 := &Producer{pi3, false, beginningOfTime} p4 := &Producer{pi1, false, beginningOfTime} db := NewRegistrationDB() // add producers db.AddProducer(Registration{"c", "a", ""}, p1) db.AddProducer(Registration{"c", "a", ""}, p2) db.AddProducer(Registration{"c", "a", "b"}, p2) db.AddProducer(Registration{"d", "a", ""}, p3) db.AddProducer(Registration{"t", "a", ""}, p4) // find producers r := db.FindRegistrations("c", "*", "").Keys() test.Equal(t, 1, len(r)) test.Equal(t, "a", r[0]) p := db.FindProducers("t", "*", "") t.Logf("%s", p) test.Equal(t, 1, len(p)) p = db.FindProducers("c", "*", "") t.Logf("%s", p) test.Equal(t, 2, len(p)) p = db.FindProducers("c", "a", "") t.Logf("%s", p) test.Equal(t, 2, len(p)) p = db.FindProducers("c", "*", "b") t.Logf("%s", p) test.Equal(t, 1, len(p)) test.Equal(t, p2.peerInfo.id, p[0].peerInfo.id) // filter by active test.Equal(t, 0, len(p.FilterByActive(sec30, sec30))) p2.peerInfo.lastUpdate = time.Now().UnixNano() test.Equal(t, 1, len(p.FilterByActive(sec30, sec30))) p = db.FindProducers("c", "*", "") t.Logf("%s", p) test.Equal(t, 1, len(p.FilterByActive(sec30, sec30))) // tombstoning fewSecAgo := time.Now().Add(-5 * time.Second).UnixNano() p1.peerInfo.lastUpdate = fewSecAgo p2.peerInfo.lastUpdate = fewSecAgo test.Equal(t, 2, len(p.FilterByActive(sec30, sec30))) p1.Tombstone() test.Equal(t, 1, len(p.FilterByActive(sec30, sec30))) time.Sleep(10 * time.Millisecond) test.Equal(t, 2, len(p.FilterByActive(sec30, 5*time.Millisecond))) // make sure we can still retrieve p1 from another registration see #148 test.Equal(t, 1, len(db.FindProducers("t", "*", "").FilterByActive(sec30, sec30))) // keys and subkeys k := db.FindRegistrations("c", "b", "").Keys() test.Equal(t, 0, len(k)) k = db.FindRegistrations("c", "a", "").Keys() test.Equal(t, 1, len(k)) test.Equal(t, "a", k[0]) k = db.FindRegistrations("c", "*", "b").SubKeys() test.Equal(t, 1, len(k)) test.Equal(t, "b", k[0]) // removing producers db.RemoveProducer(Registration{"c", "a", ""}, p1.peerInfo.id) p = db.FindProducers("c", "*", "*") t.Logf("%s", p) test.Equal(t, 1, len(p)) db.RemoveProducer(Registration{"c", "a", ""}, p2.peerInfo.id) db.RemoveProducer(Registration{"c", "a", "b"}, p2.peerInfo.id) p = db.FindProducers("c", "*", "*") t.Logf("%s", p) test.Equal(t, 0, len(p)) // do some key removals k = db.FindRegistrations("c", "*", "*").Keys() test.Equal(t, 2, len(k)) db.RemoveRegistration(Registration{"c", "a", ""}) db.RemoveRegistration(Registration{"c", "a", "b"}) k = db.FindRegistrations("c", "*", "*").Keys() test.Equal(t, 0, len(k)) } func fillRegDB(registrations int, producers int) *RegistrationDB { regDB := NewRegistrationDB() for i := 0; i < registrations; i++ { regT := Registration{"topic", "t" + strconv.Itoa(i), ""} regCa := Registration{"channel", "t" + strconv.Itoa(i), "ca" + strconv.Itoa(i)} regCb := Registration{"channel", "t" + strconv.Itoa(i), "cb" + strconv.Itoa(i)} for j := 0; j < producers; j++ { p := Producer{ peerInfo: &PeerInfo{ id: "p" + strconv.Itoa(j), }, } regDB.AddProducer(regT, &p) regDB.AddProducer(regCa, &p) regDB.AddProducer(regCb, &p) } } return regDB } func benchmarkLookupRegistrations(b *testing.B, registrations int, producers int) { regDB := fillRegDB(registrations, producers) b.ResetTimer() for i := 0; i < b.N; i++ { j := strconv.Itoa(rand.Intn(producers)) _ = regDB.LookupRegistrations("p" + j) } } func benchmarkRegister(b *testing.B, registrations int, producers int) { for i := 0; i < b.N; i++ { _ = fillRegDB(registrations, producers) } } func benchmarkDoLookup(b *testing.B, registrations int, producers int) { regDB := fillRegDB(registrations, producers) b.ResetTimer() for i := 0; i < b.N; i++ { topic := "t" + strconv.Itoa(rand.Intn(registrations)) _ = regDB.FindRegistrations("topic", topic, "") _ = regDB.FindRegistrations("channel", topic, "*").SubKeys() _ = regDB.FindProducers("topic", topic, "") } } func BenchmarkLookupRegistrations8x8(b *testing.B) { benchmarkLookupRegistrations(b, 8, 8) } func BenchmarkLookupRegistrations8x64(b *testing.B) { benchmarkLookupRegistrations(b, 8, 64) } func BenchmarkLookupRegistrations64x64(b *testing.B) { benchmarkLookupRegistrations(b, 64, 64) } func BenchmarkLookupRegistrations64x512(b *testing.B) { benchmarkLookupRegistrations(b, 64, 512) } func BenchmarkLookupRegistrations512x512(b *testing.B) { benchmarkLookupRegistrations(b, 512, 512) } func BenchmarkLookupRegistrations512x2048(b *testing.B) { benchmarkLookupRegistrations(b, 512, 2048) } func BenchmarkRegister8x8(b *testing.B) { benchmarkRegister(b, 8, 8) } func BenchmarkRegister8x64(b *testing.B) { benchmarkRegister(b, 8, 64) } func BenchmarkRegister64x64(b *testing.B) { benchmarkRegister(b, 64, 64) } func BenchmarkRegister64x512(b *testing.B) { benchmarkRegister(b, 64, 512) } func BenchmarkRegister512x512(b *testing.B) { benchmarkRegister(b, 512, 512) } func BenchmarkRegister512x2048(b *testing.B) { benchmarkRegister(b, 512, 2048) } func BenchmarkDoLookup8x8(b *testing.B) { benchmarkDoLookup(b, 8, 8) } func BenchmarkDoLookup8x64(b *testing.B) { benchmarkDoLookup(b, 8, 64) } func BenchmarkDoLookup64x64(b *testing.B) { benchmarkDoLookup(b, 64, 64) } func BenchmarkDoLookup64x512(b *testing.B) { benchmarkDoLookup(b, 64, 512) } func BenchmarkDoLookup512x512(b *testing.B) { benchmarkDoLookup(b, 512, 512) } func BenchmarkDoLookup512x2048(b *testing.B) { benchmarkDoLookup(b, 512, 2048) } ================================================ FILE: nsqlookupd/tcp.go ================================================ package nsqlookupd import ( "io" "net" "sync" "github.com/nsqio/nsq/internal/protocol" ) type tcpServer struct { nsqlookupd *NSQLookupd conns sync.Map } func (p *tcpServer) Handle(conn net.Conn) { p.nsqlookupd.logf(LOG_INFO, "TCP: new client(%s)", conn.RemoteAddr()) // The client should initialize itself by sending a 4 byte sequence indicating // the version of the protocol that it intends to communicate, this will allow us // to gracefully upgrade the protocol away from text/line oriented to whatever... buf := make([]byte, 4) _, err := io.ReadFull(conn, buf) if err != nil { p.nsqlookupd.logf(LOG_ERROR, "failed to read protocol version - %s", err) conn.Close() return } protocolMagic := string(buf) p.nsqlookupd.logf(LOG_INFO, "CLIENT(%s): desired protocol magic '%s'", conn.RemoteAddr(), protocolMagic) var prot protocol.Protocol switch protocolMagic { case " V1": prot = &LookupProtocolV1{nsqlookupd: p.nsqlookupd} default: protocol.SendResponse(conn, []byte("E_BAD_PROTOCOL")) conn.Close() p.nsqlookupd.logf(LOG_ERROR, "client(%s) bad protocol magic '%s'", conn.RemoteAddr(), protocolMagic) return } client := prot.NewClient(conn) p.conns.Store(conn.RemoteAddr(), client) err = prot.IOLoop(client) if err != nil { p.nsqlookupd.logf(LOG_ERROR, "client(%s) - %s", conn.RemoteAddr(), err) } p.conns.Delete(conn.RemoteAddr()) client.Close() } func (p *tcpServer) Close() { p.conns.Range(func(k, v interface{}) bool { v.(protocol.Client).Close() return true }) } ================================================ FILE: test.sh ================================================ #!/bin/sh set -e GOMAXPROCS=1 go test -timeout 90s ./... if [ "$GOARCH" = "amd64" ] || [ "$GOARCH" = "arm64" ]; then # go test: -race is only supported on linux/amd64, linux/ppc64le, # linux/arm64, freebsd/amd64, netbsd/amd64, darwin/amd64 and windows/amd64 GOMAXPROCS=4 go test -timeout 90s -race ./... fi # no tests, but a build is something for dir in apps/*/ bench/*/; do dir=${dir%/} if grep -q '^package main$' $dir/*.go 2>/dev/null; then echo "building $dir" go build -o $dir/$(basename $dir) ./$dir else echo "(skipped $dir)" fi done # disable "composite literal uses unkeyed fields" go vet -composites=false ./... FMTDIFF="$(find apps internal nsqd nsqlookupd -name '*.go' -exec gofmt -d '{}' ';')" if [ -n "$FMTDIFF" ]; then printf '%s\n' "$FMTDIFF" exit 1 fi