Showing preview only (1,244K chars total). Download the full file or copy to clipboard to get everything.
Repository: nsqio/nsq
Branch: master
Commit: 9ea507128926
Files: 211
Total size: 1.2 MB
Directory structure:
gitextract_w7hjwstt/
├── .github/
│ └── workflows/
│ └── test.yml
├── .gitignore
├── AUTHORS
├── CODE_OF_CONDUCT.md
├── CONTRIBUTING.md
├── ChangeLog.md
├── Dockerfile
├── LICENSE
├── Makefile
├── README.md
├── apps/
│ ├── nsq_stat/
│ │ └── nsq_stat.go
│ ├── nsq_tail/
│ │ └── nsq_tail.go
│ ├── nsq_to_file/
│ │ ├── file_logger.go
│ │ ├── nsq_to_file.go
│ │ ├── options.go
│ │ ├── strftime.go
│ │ └── topic_discoverer.go
│ ├── nsq_to_http/
│ │ ├── http.go
│ │ ├── nsq_to_http.go
│ │ └── nsq_to_http_test.go
│ ├── nsq_to_nsq/
│ │ └── nsq_to_nsq.go
│ ├── nsqadmin/
│ │ ├── main.go
│ │ ├── main_test.go
│ │ └── options.go
│ ├── nsqd/
│ │ ├── README.md
│ │ ├── main.go
│ │ ├── main_test.go
│ │ └── options.go
│ ├── nsqlookupd/
│ │ ├── README.md
│ │ ├── main.go
│ │ ├── main_test.go
│ │ └── options.go
│ └── to_nsq/
│ ├── README.md
│ └── to_nsq.go
├── bench/
│ ├── bench.py
│ ├── bench_channels/
│ │ └── bench_channels.go
│ ├── bench_reader/
│ │ └── bench_reader.go
│ ├── bench_writer/
│ │ └── bench_writer.go
│ └── requirements.txt
├── bench.sh
├── contrib/
│ ├── nsq.spec
│ ├── nsqadmin.cfg.example
│ ├── nsqd.cfg.example
│ └── nsqlookupd.cfg.example
├── coverage.sh
├── dist.sh
├── fmt.sh
├── go.mod
├── go.sum
├── internal/
│ ├── app/
│ │ ├── float_array.go
│ │ └── string_array.go
│ ├── auth/
│ │ └── authorizations.go
│ ├── clusterinfo/
│ │ ├── data.go
│ │ ├── producer_test.go
│ │ └── types.go
│ ├── dirlock/
│ │ ├── dirlock.go
│ │ ├── dirlock_illumos.go
│ │ └── dirlock_windows.go
│ ├── http_api/
│ │ ├── api_request.go
│ │ ├── api_response.go
│ │ ├── compress.go
│ │ ├── http_server.go
│ │ ├── req_params.go
│ │ └── topic_channel_args.go
│ ├── lg/
│ │ ├── lg.go
│ │ └── lg_test.go
│ ├── pqueue/
│ │ ├── pqueue.go
│ │ └── pqueue_test.go
│ ├── protocol/
│ │ ├── byte_base10.go
│ │ ├── byte_base10_test.go
│ │ ├── errors.go
│ │ ├── names.go
│ │ ├── protocol.go
│ │ └── tcp_server.go
│ ├── quantile/
│ │ ├── aggregate.go
│ │ └── quantile.go
│ ├── statsd/
│ │ ├── client.go
│ │ └── host.go
│ ├── stringy/
│ │ ├── slice.go
│ │ ├── slice_test.go
│ │ └── template.go
│ ├── test/
│ │ ├── assertions.go
│ │ ├── fakes.go
│ │ └── logger.go
│ ├── util/
│ │ ├── rand.go
│ │ ├── unix_socket.go
│ │ ├── util_test.go
│ │ └── wait_group_wrapper.go
│ ├── version/
│ │ └── binary.go
│ └── writers/
│ ├── boundary_buffered_writer.go
│ └── spread_writer.go
├── nsqadmin/
│ ├── .eslintrc
│ ├── README.md
│ ├── gulp
│ ├── gulpfile.js
│ ├── http.go
│ ├── http_test.go
│ ├── logger.go
│ ├── notify.go
│ ├── nsqadmin.go
│ ├── nsqadmin_test.go
│ ├── options.go
│ ├── package.json
│ ├── static/
│ │ ├── build/
│ │ │ ├── base.css
│ │ │ ├── index.html
│ │ │ ├── main.js
│ │ │ └── vendor.js
│ │ ├── css/
│ │ │ └── base.scss
│ │ ├── html/
│ │ │ └── index.html
│ │ └── js/
│ │ ├── app_state.js
│ │ ├── collections/
│ │ │ ├── nodes.js
│ │ │ └── topics.js
│ │ ├── lib/
│ │ │ ├── ajax_setup.js
│ │ │ ├── handlebars_helpers.js
│ │ │ └── pubsub.js
│ │ ├── main.js
│ │ ├── models/
│ │ │ ├── channel.js
│ │ │ ├── node.js
│ │ │ └── topic.js
│ │ ├── router.js
│ │ └── views/
│ │ ├── app.js
│ │ ├── base.js
│ │ ├── channel.hbs
│ │ ├── channel.js
│ │ ├── counter.hbs
│ │ ├── counter.js
│ │ ├── error.hbs
│ │ ├── header.hbs
│ │ ├── header.js
│ │ ├── lookup.hbs
│ │ ├── lookup.js
│ │ ├── node.hbs
│ │ ├── node.js
│ │ ├── nodes.hbs
│ │ ├── nodes.js
│ │ ├── spinner.hbs
│ │ ├── topic.hbs
│ │ ├── topic.js
│ │ ├── topics.hbs
│ │ ├── topics.js
│ │ └── warning.hbs
│ ├── static.go
│ └── test/
│ ├── ca.key
│ ├── ca.pem
│ ├── ca.srl
│ ├── cert.pem
│ ├── client.key
│ ├── client.pem
│ ├── client.req
│ ├── key.pem
│ ├── server.key
│ ├── server.pem
│ └── server.req
├── nsqd/
│ ├── README.md
│ ├── backend_queue.go
│ ├── buffer_pool.go
│ ├── channel.go
│ ├── channel_test.go
│ ├── client_v2.go
│ ├── dqname.go
│ ├── dqname_windows.go
│ ├── dummy_backend_queue.go
│ ├── guid.go
│ ├── guid_test.go
│ ├── http.go
│ ├── http_test.go
│ ├── in_flight_pqueue.go
│ ├── in_flight_pqueue_test.go
│ ├── logger.go
│ ├── lookup.go
│ ├── lookup_peer.go
│ ├── message.go
│ ├── nsqd.go
│ ├── nsqd_test.go
│ ├── options.go
│ ├── protocol_v2.go
│ ├── protocol_v2_test.go
│ ├── protocol_v2_unixsocket_test.go
│ ├── stats.go
│ ├── stats_test.go
│ ├── statsd.go
│ ├── tcp.go
│ ├── test/
│ │ ├── cert.sh
│ │ ├── certs/
│ │ │ ├── ca.key
│ │ │ ├── ca.pem
│ │ │ ├── ca.srl
│ │ │ ├── cert.pem
│ │ │ ├── client.key
│ │ │ ├── client.pem
│ │ │ ├── client.req
│ │ │ ├── key.pem
│ │ │ ├── server.key
│ │ │ ├── server.pem
│ │ │ └── server.req
│ │ └── openssl.conf
│ ├── topic.go
│ └── topic_test.go
├── nsqlookupd/
│ ├── README.md
│ ├── client_v1.go
│ ├── http.go
│ ├── http_test.go
│ ├── logger.go
│ ├── lookup_protocol_v1.go
│ ├── lookup_protocol_v1_test.go
│ ├── nsqlookupd.go
│ ├── nsqlookupd_test.go
│ ├── options.go
│ ├── registration_db.go
│ ├── registration_db_test.go
│ └── tcp.go
└── test.sh
================================================
FILE CONTENTS
================================================
================================================
FILE: .github/workflows/test.yml
================================================
name: tests
on:
push: {branches: [master]}
pull_request: {branches: [master]}
jobs:
test:
runs-on: ubuntu-20.04
timeout-minutes: 30
strategy:
fail-fast: false
matrix:
go: ["1.21.x", "1.22.x", "1.23.x"]
arch: ["amd64", "386"]
env:
GOARCH: "${{matrix.arch}}"
steps:
- uses: actions/checkout@v2
- uses: WillAbides/setup-go-faster@v1.7.0
with:
go-version: ${{matrix.go}}
- name: build
run: make all
- name: test
run: ./test.sh
staticcheck:
runs-on: ubuntu-20.04
steps:
- uses: actions/checkout@v2
- uses: dominikh/staticcheck-action@v1.3.1
with:
version: "2024.1.1"
install-go: false
code-coverage:
runs-on: ubuntu-20.04
steps:
- uses: actions/checkout@v2
- name: install goveralls
run: go install github.com/mattn/goveralls@latest
- name: send coverage
env:
COVERALLS_TOKEN: ${{secrets.GITHUB_TOKEN}}
run: ./coverage.sh --coveralls
================================================
FILE: .gitignore
================================================
/build/
dist/
.cover/
profile/
# nsqd data from testing
*.dat
# nsqadmin
node_modules
# apps
apps/nsqlookupd/nsqlookupd
apps/nsqd/nsqd
apps/nsqadmin/nsqadmin
apps/nsq_to_nsq/nsq_to_nsq
apps/nsq_to_file/nsq_to_file
apps/nsq_pubsub/nsq_pubsub
apps/nsq_to_http/nsq_to_http
apps/nsq_tail/nsq_tail
apps/nsq_stat/nsq_stat
apps/to_nsq/to_nsq
bench/bench_reader/bench_reader
bench/bench_writer/bench_writer
bench/bench_channels/bench_channels
# Go.gitignore
# Compiled Object files, Static and Dynamic libs (Shared Objects)
*.o
*.a
*.so
# Folders
_obj
_test
# Architecture specific extensions/prefixes
*.[568vq]
[568vq].out
*.cgo1.go
*.cgo2.c
_cgo_defun.c
_cgo_gotypes.go
_cgo_export.*
_testmain.go
*.exe
# vim stuff
*.sw[op]
================================================
FILE: AUTHORS
================================================
# For a complete listing, see https://github.com/nsqio/nsq/graphs/contributors
# Original Authors
Matt Reiferson <mreiferson@gmail.com>
Jehiah Czebotar <jehiah@gmail.com>
# Maintainers
Pierce Lopez <ploxiln@gmail.com>
# Disclaimer
Matt Reiferson's contributions to this project are being made solely in a personal capacity
and does not convey any rights to any intellectual property of any third parties.
================================================
FILE: CODE_OF_CONDUCT.md
================================================
# Contributor Code of Conduct
As contributors and maintainers of this project, and in the interest of fostering an open and
welcoming community, we pledge to respect all people who contribute through reporting issues,
posting feature requests, updating documentation, submitting pull requests or patches, and other
activities.
We are committed to making participation in this project a harassment-free experience for everyone,
regardless of level of experience, gender, gender identity and expression, sexual orientation,
disability, personal appearance, body size, race, ethnicity, age, religion, or nationality.
Examples of unacceptable behavior by participants include:
* The use of sexualized language or imagery
* Personal attacks
* Trolling or insulting/derogatory comments
* Public or private harassment
* Publishing other's private information, such as physical or electronic addresses, without explicit permission
* Other unethical or unprofessional conduct.
Project maintainers have the right and responsibility to remove, edit, or reject comments, commits,
code, wiki edits, issues, and other contributions that are not aligned to this Code of Conduct. By
adopting this Code of Conduct, project maintainers commit themselves to fairly and consistently
applying these principles to every aspect of managing this project. Project maintainers who do not
follow or enforce the Code of Conduct may be permanently removed from the project team.
This code of conduct applies both within project spaces and in public spaces when an individual is
representing the project or its community.
Instances of abusive, harassing, or otherwise unacceptable behavior may be reported by opening an
issue or contacting one or more of the project maintainers.
This Code of Conduct is adapted from the [Contributor Covenant][1], version 1.2.0, available at
[http://contributor-covenant.org/version/1/2/0/][2].
[1]: http://contributor-covenant.org
[2]: http://contributor-covenant.org/version/1/2/0/
================================================
FILE: CONTRIBUTING.md
================================================
# Contributing
Thanks for your interest in contributing to NSQ!
## Code of Conduct
Help us keep NSQ open and inclusive. Please read and follow our [Code of Conduct](CODE_OF_CONDUCT.md).
## Getting Started
* make sure you have a [GitHub account](https://github.com/signup/free)
* submit a ticket for your issue, assuming one does not already exist
* clearly describe the issue including steps to reproduce when it is a bug
* identify specific versions of the binaries and client libraries
* fork the repository on GitHub
## Making Changes
* create a branch from where you want to base your work
* we typically name branches according to the following format: `helpful_name_<issue_number>`
* make commits of logical units
* make sure your commit messages are in a clear and readable format, example:
```
nsqd: fixed bug in protocol_v2
* update the message pump to properly account for RDYness
* cleanup variable names
* ...
```
* if you're fixing a bug or adding functionality it probably makes sense to write a test
* make sure to run `fmt.sh` and `test.sh` in the root of the repo to ensure that your code is
properly formatted and that tests pass (we use GitHub Actions for continuous integration)
## Submitting Changes
* push your changes to your branch in your fork of the repository
* submit a pull request against nsqio's repository
* comment in the pull request when you're ready for the changes to be reviewed: `"ready for review"`
================================================
FILE: ChangeLog.md
================================================
# NSQ Changelog
## Releases
### 1.3.0 - 2023-12-26
**Upgrading**
* #1427 / #1373 / #1371 - fix staticcheck warnings, remove support for gobindata / go 1.16
Features:
* #1473 - `nsqd`: use --tls-root-ca-file in nsqauth request (thanks @intellitrend-team)
* #1470 / #1469 - `nsqadmin`: upgrade supported ECMA from ES5 to ES2020 (thanks @dudleycarr)
* #1468 - `nsqadmin`: add paused label to topic within the node view (thanks @dudleycarr)
* #1462 - `nsqadmin`: add admin check for topic/node thombstone endpoint (thanks @dudleycarr)
* #1434 - `nsqd`: add support of unix sockets for tcp, http, https listeners (thanks @telepenin)
* #1424 - `nsqd`: add /debug/freememory API (thanks @guozhao-coder)
* #1421 - `nsqd`: nicer tls-min-version help text default
* #1376 - `nsqd`: allow unbuffered memory chan if ephemeral or deferred
* #1380 - `nsqd`: use metadata struct for both marshal and unmarshal (thanks @karalabe)
* #1403 - `nsqd`: /info api returns more info (thanks @arshabbir)
* #1384 - `nsqd`: allow disabling both HTTP and HTTPS interfaces (thanks @karalabe)
* #1385 - `nsqd`: enable support for TLS1.3 (thanks @karalabe)
* #1372 - `nsqadmin`: new flag --dev-static-dir instead of debug build tag
Bugs:
* #1478 - `Dockerfile`: remove nsswitch.conf check (thanks @dudleycarr)
* #1467 - `nsqadmin`: fix counter by bounding animation steps (thanks @dudleycarr)
* #1466 - `nsqadmin`: fix broken graph template in nsqadmin node view (thanks @dudleycarr)
* #1455 / #1387 - update dependencies
* #1445 - `nsqd`: fix unsafe concurrency read in RemoveClient (thanks @gueFDF)
* #1441 - `nsqd`: fix panic when statsd enabled and memstats disabled with no topics (thanks @carl-reverb)
* #1428 - delete `validTopicChannelNameRegex` useless escape characters (thanks @sjatsh)
* #1419 - contrib: update nsqadmin.cfg.example (thanks @StellarisW)
### 1.2.1 - 2021-08-15
**Upgrading**
* #1227 - bump dependencies, officially drop `dep` support, drop Go `1.9` support
Features:
* #1347 - `nsqadmin`: switch to go:embed for static assets
* #1355 / #1364 - arm64 builds (thanks @danbf)
* #1346 - `nsqd`: ability to skip ephemeral topics/channels in statsd output
* #1336 / #1341 / #1343 - `nsqd`: ability to configure auth endpoint path (thanks @tufanbarisyildirim)
* #1307 - remove `Context` to use stdlib `context`
* #1295 / #1296 - switch to GitHub Actions CI
* #1292 - `nsqd`: minimize allocations on message send (thanks @imxyb)
* #1289 - optimize `uniq` (thanks @donutloop)
* #1230 / #1232 - `nsqd`: ability to omit memory stats from `/stats` (thanks @creker)
* #1226 - `nsqd`: only update internal `RDY` count for client when it changes (thanks @andyxning)
* #1221 / #1363 - test against more recent versions of Go
* #1209 - `nsqd`: bump `go-diskqueue` (interface change) (thanks @bitpeng)
* #1206 - prefer idiomatic `sort.Ints` over `sort.Sort` (thanks @lelenanam)
* #1197 / #1362 - Dockerfile: update Alpine base image, use /data by default
* #1178 - `nsqd`: configurable queue scan worker pool (thanks @andyxning)
* #1159 - `nsqd`: don't buffer messages when `--mem-queue-size=0` (thanks @bitpeng)
* #1073 / #1297 - `nsqd`: support separate broadcast ports for TCP and HTTP (thanks @shyam-king)
Bugs:
* #1347 - `nsqadmin`: fix graphite key for ephemeral topics/channels
* #765 / #1195 / #1203 / #1205 - fix build on illumos (thanks @i-sevostyanov)
* #1333 - fix race detector tests on non-bash shells
* #1330 - fix `log_level` support in configuration file (thanks @edoger)
* #1319 / #1331 / #1361 - `nsqd`: handle SIGTERM
* #1287 - `nsqadmin`: fix `--proxy-graphite` support (thanks @fanlix)
* #1270 / #1271 - `nsqlookupd`: fix incorrect error message for HTTP listener (thanks @TangDH03)
* #1264 - fix benchmark script
* #1251 / #1314 / #1327 - `nsqd`: fix live lock for high churn ephemeral topic/channel reconnections (thanks @slayercat)
* #1237 - Dockerfile: add `nsswitch.conf` to ensure go resolver uses `/etc/hosts` first
* #1217 / #1220 - `nsqd`: improve error message when `--data-path` does not exist (thanks @mdh67899)
* #1198 / #1190 / #1262 - synchronize close of all connections on Exit (thanks @benjsto)
* #1188 / #1189 - `nsqadmin`: fix channel delete, fix source-maps in Firefox (thanks @avtrifanov)
* #1186 - `nsqadmin`: fix nodes list with ipv6 addresses (thanks @andyxning)
### 1.2.0 - 2019-08-26
**Upgrading**
* #1055 - `nsqd`: removed support for old metadata scheme used in v0.3.8 and earlier
* you cannot upgrade directly from v0.3.8 to v1.2.0, you must go through v1.0.0-compat or v1.1.0
* #1115 - manage dependencies with go modules
* `dep` support still present for now, but deprecated
Features:
* #1136 - `nsqd`: add `--max-channel-consumers` (default unlimited) (thanks @andyxning)
* #1133 - `nsqd`: add `--min-output-buffer-timeout` (default 25ms) to limit how low a timeout a consumer can request
* and raise default `--max-output-buffer-timeout` to 30 seconds (lower timeout, more cpu usage)
* #1127 - `nsqd`: add topic total message bytes to stats (thanks @andyxning)
* #1125 - `nsqd`: add flag to adjust default `--output-buffer-timeout` (thanks @andyxning)
* #1163 - `nsqd`: add random load balancing for authd requests (thanks @shenhui0509)
* #1119 - `nsqd`: include client TLS cert CommonName in authd requests
* #1147 - `nsq_to_file`: include topic/channel in most log messages
* #1117 - `nsq_to_file`: add `--log-level` and `--log-prefix` flags
* #1117/#1120/#1123 - `nsq_to_file`: big refactor, more robust file switching and syncing and error handling
* #1118 - `nsqd`: add param to `/stats` endpoint to allow skipping per-client stats (much faster if many clients)
* #1118 - `nsqadmin`, `nsq_stat`: use `include_clients` param for `/stats` for a big speedup for big clusters
* #1110 - `nsq_to_file`: support for separate working directory with `--work-dir` (thanks @mccutchen)
* #856 - `nsqadmin`: add `--base-path` flag (thanks @blinklv)
* #1072 - `nsq_to_http`: add `--header` flag (thanks @alwindoss)
* #881 - `nsqd`: add producer client tcp connections to stats (thanks @sparklxb)
* #1071/#1074 - `nsq_to_file`: new flag `--sync-interval` (default same as previous behavior, 30 seconds) (thanks @alpaker)
Bugs:
* #1153 - `nsqd`: close connections that don't send "magic" header (thanks @JoseFeng)
* #1140 - `nsqd`: exit on all fatal Accept() errors - restart enables better recovery for some conditions (thanks @mdh67899)
* #1140 - `nsqd`, `nsqlookupd`, `nsqadmin`: refactor LogLevel, general refactor to better exit on all fatal errors
* #1140 - `nsqadmin`: switch to using `judwhite/go-svc` like `nsqd` and `nsqadmin` do
* #1134 - `nsqadmin`: fix clients count and channel total message rate (new bugs introduced in this cycle)
* #1132 - `nsqd`, `nsqlookupd`, `nsqadmin`: fix http error response unreliable json serialization
* #1116 - `nsqlookupd`: fix orphaned ephemeral topics in registration DB
* #1109 - `nsqd`: fix topic message mis-counting if there are backend write errors (thanks @SwanSpouse)
* #1099 - `nsqlookupd`: optimize `/nodes` endpoint, much better for hundreds of nsqd (thanks @andyxning)
* #1085 - switch `GOFLAGS` to `BLDFLAGS` in `Makefile` now that `GOFLAGS` is automatically used by go
* #1080 - `nsqadmin`: eslint reported fixes/cleanups
### 1.1.0 - 2018-08-19
**Upgrading from 1.0.0-compat**: Just a few backwards incompatible changes:
* #1056 - Removed the `nsq_pubsub` utility
* #873 - `nsqd` flags `--msg-timeout` and `--statsd-interval` only take duration strings
* plain integer no longer supported
* #921 - `nsqd`: http `/mpub` endpoint `binary` param interprets "0" or "false" to mean text mode
* previously any value meant to use binary mode instead of text mode - (thanks @andyxning)
The previous release, version "1.0.0-compat", was curiously-named to indicate an almost
(but not quite) complete transition to a 1.0 api-stable release line. Confusingly, this
follow-up release which completes the transition comes more than a year later. Because there
have been a fair number of changes and improvements in the past year, an additional minor
version bump seems appropriate.
Features:
* #874 - `nsqd`: add memory stats to http `/stats` response (thanks @sparklxb)
* #892 - `nsqd`, `nsqlookupd`, `nsqadmin`: add `--log-level` option (deprecating `--verbose`) (thanks @antihax)
* #898 - `nsqd`, `nsqlookupd`, `nsqadmin`: logging refactor to use log levels everywhere
* #914 - `nsqadmin`: `X-Forwarded-User` based "admin" permission (thanks @chen-anders)
* #929 - `nsqd`: add topic/channel filter to `/stats`, use in `nsqadmin` and `nsq_stat` for efficiency (thanks @andyxning)
* #936 - `nsq_to_file`: refactor/cleanup
* #945 - `nsq_to_nsq`: support multiple `--topic` flags (thanks @jlr52)
* #957 - `nsq_tail`: support multiple `--topic` flags (thanks @soar)
* #946 - `nsqd`, `nsqadmin`: update internal http client with new go `http.Transport` features (keepalives, timeouts, dualstack)
* affects metadata/stats requests between `nsqadmin`, `nsqd`, `nsqlookupd`
* #954 - manage dependencies with `dep` (replacing `gpm`) (thanks @judwhite)
* #957 - multi-stage docker image build (thanks @soar)
* #996 - `nsqd`: better memory usage when messages have different sizes (thanks @andyxning)
* #1019 - `nsqd`: optimize random channel selection in queueScanLoop (thanks @vearne)
* #1025 - `nsqd`: buffer and spread statsd udp sends (avoid big burst of udp, less chance of loss)
* #1038 - `nsqlookupd`: optimize for many producers (thousands) (thanks @andyxning)
* #1050/#1053 - `nsqd`: new topic can be unlocked faster after creation
* #1062 - `nsqadmin`: update JS deps
Bugs:
* #753 - `nsqadmin`: fix missing channels in topic list
* #867 - `to_nsq`: fix divide-by-zero issue when `--rate` not specified (thanks @adamweiner)
* #868 - `nsqd`: clamp requeue timeout to range instead of dropping connection (thanks @tsholmes)
* #891 - `nsqd`: fix race when client subscribes to ephemeral topic or channel while it is being cleaned up (reported by @slayercat)
* #927 - `nsqd`: fix deflate level handling
* #934 - `nsqd`: fix channel shutdown flush race
* #935 - `nsq_to_file`: fix connection leaks when using `--topic-pattern` (thanks @jxskiss)
* #951 - mention docker images and binaries for additional platforms in README (thanks @DAXaholic)
* #950 - `nsqlookupd`: close connection when magic read fails (thanks @yc90s)
* #971 - `nsqd`: fix some races getting ChannelStats (thanks @daroot)
* #988 - `nsqd`: fix e2e timings config example, add range validation (thanks @protoss-player)
* #991 - `nsq_tail`: logging to stderr (only nsq messages to stdout)
* #1000 - `nsq_to_http`: fix http connect/request timeout flags (thanks @kamyanskiy)
* #993/#1008 - `nsqd`: fix possible lookupd-identify-error busy-loop (reported by @andyxning)
* #1005 - `nsqadmin`: fix typo "Delfate" in connection attrs list (thanks @arussellsaw)
* #1032 - `nsqd`: fix loading metadata with messages queued on un-paused topic with multiple channels (thanks @michaelyou)
* #1004 - `nsqlookupd`: exit with error when failed to listen on ports (thanks @stephens2424)
* #1068 - `nsqadmin`: fix html escaping for large_graph url inside javascript
* misc test suite improvements and updates (go versions, tls certs, ...)
### 1.0.0-compat - 2017-03-21
**Upgrading from 0.3.8**: Numerous backwards incompatible changes:
* Deprecated `nsqd` features removed:
* Pre-V1 HTTP endpoints / response format:
* `/{m,}put` (use `/{m,}pub`)
* `/{create,delete,empty,pause,unpause}_{topic,channel}` (use `/{topic,channel}/<operation>`)
* `--max-message-size` flag (use `--max-msg-size`)
* `V2` protocol `IDENTIFY` command `short_id` and `long_id` properties (use `client_id`, `hostname`, and `user_agent`)
* `/stats` HTTP response `name` property (use `client_id`)
* Deprecated `nsqlookupd` features removed:
* Pre-V1 HTTP endpoints / response format:
* `/{create,delete}_{topic,channel}` (use `/{topic,channel}/<operation>`)
* `/tombstone_topic_producer` (use `/topic/tombstone`)
* Deprecated `nsqadmin` features removed:
* `--template-dir` flag (not required, templates are compiled into binary)
* `--use-statsd-prefixes` flag (use `--statsd-counter-format` and `--statsd-gauge-format`)
* `nsq_stat` `--status-every` flag (use `--interval`)
* `--reader-opt` on all binaries that had this flag (use `--consumer-opt`)
* `nsq_to_file` `--gzip-compression` flag (use `--gzip-level`)
* `nsq_to_http` `--http-timeout` and `--http-timeout-ms` flags (use `--http-connect-timeout` and `--http-request-timeout`)
* `nsq_to_http` `--round-robin` flag (use `--mode=round-robin`)
* `nsq_to_http` `--max-backoff-duration` flag (use `--consumer-opt=max_backoff_duration,X`)
* `nsq_to_http` `--throttle-fraction` flag (use `--sample=X`)
* `nsq_to_nsq` `--max-backoff-duration` flag (use `--consumer-opt=max_backoff_duration,X`)
* `nsqd` `--worker-id` deprecated in favor of `--node-id` (to be fully removed in subsequent release)
This is a compatibility release that drops a wide range of previously deprecated features (#367)
while introducing some new deprecations (#844) that we intend to fully remove in a subsequent 1.0
release.
Of note, all of the pre-1.0 HTTP endpoints (and response formats) are gone. Any clients or tools
that use these endpoints/response formats won't work with this release. These changes have been
available since 0.2.29 (released in July of 2014). Clients wishing to forwards-compatibly upgrade
can either use the new endpoints or send the following header:
Accept: application/vnd.nsq version=1.0
Also, many command line flags have been removed — in almost all cases an alternative is available
with a (hopefully) more obvious name. These changes have the same affect on config file option
names.
On Linux, this release will automatically migrate `nsq.<worker-id>.dat` named metadata files to
`nsq.dat` in a way that allows users to seamlessly _downgrade_ from this release back to 0.3.8, if
necessary. A subsequent release will clean up these convenience symlinks and observe only
`nsq.dat`. See the discussion in #741 and the changes #844 for more details.
Performance wise, #741 landed which significantly reduces global contention on internal message ID
generation, providing a ~1.75x speed improvement on multi-topic benchmarks.
Finally, a number of minor issues were resolved spanning contributions from 9 community members!
Thanks!
Features:
* #766 - use `alpine` base image for official Docker container (thanks @kenjones-cisco)
* #775 - `nsqadmin:` `/config` API (thanks @kenjones-cisco)
* #776 - `nsqadmin`, `nsq_stat`, `nsq_to_file`, `nsq_to_http`: HTTP client connect/request timeouts (thanks @kenjones-cisco)
* #777/#778/#783/#785 - improve test coverage (thanks @kenjones-cisco)
* #788 - `to_nsq`: add `--rate` message throttling option
* #367 - purge deprecated features (see above)
* #741 - `nsqd`: per-topic message IDs (multi-topic pub benchmarks up to ~1.75x faster)
* #850 - `nsqd`, `nsqlookupd`, `nsqadmin`: add `--log-prefix` option (thanks @ploxiln)
* #844 - `nsqd`: deprecate `--worker-id` for `--node-id` and drop ID from `nsqd.dat` file (thanks @ploxiln)
Bugs:
* #787 - `nsqlookupd`: properly close TCP connection in `IOLoop` (thanks @JackDrogon)
* #792 - `nsqdmin`: fix root CA verification (thanks @joshuarubin)
* #794 - `nsq_to_file`: require `--topic` or `--topic-pattern` (thanks @judwhite)
* #816/#823 - `nsqadmin`: fix handling of IPv6 broadcast addresses (thanks @magnetised)
* #805/#832 - `nsqd`: fix requeue and deferred message accounting (thanks @sdbaiguanghe)
* #532/#830 - `nsqd`: switch to golang/snappy to fix snappy deadlock
* #826/#831/#837/#839 - `nsqd`: fix default `--broadcast-address` and error when `nsqlookupd` reqs fail (thanks @ploxiln @stephensearles)
* #822/#835 - `nsqd`: prevent panic in binary `/mpub` (thanks @yangyifeng01)
* #841 - `nsqadmin`: allow ctrl/meta+click to open a new tab
* #843 - `nsqd`: check for exit before requeing
### 0.3.8 - 2016-05-26
**Upgrading from 0.3.7**: Binaries contain no backwards incompatible changes.
This release fixes a critical regression in `0.3.7` that could result in message loss when
attempting to cleanly shutdown `nsqd` by sending it a `SIGTERM`. The expected behavior was for it
to flush messages in internal buffers to disk before exiting. See #757 and #759 for more details.
A few performance improvements landed including #743, which improves channel throughput by ~17%,
and #740, which reduces garbage when reading messages from disk.
We're now stripping debug info, reducing binary size, in the official binary downloads and Windows
binaries are now bundled with the appropriate `.exe` extension (#726 and #751).
Features:
* #743 - `nsqd`: remove channel `messagePump`
* #751 - strip debug info from binaries (thanks @ploxiln)
* #740 - `nsqd`: reduce garbage when reading from diskqueue (thanks @dieterbe)
Bugs:
* #757/#759 - `nsqd`: properly handle `SIGTERM` (thanks @judwhite)
* #738 - updates for latest `go-options`
* #730 - `nsqd`: diskqueue sync count on both read/write
* #734 - `nsqadmin`: make `rate` column work without `--proxy-graphite` (thanks @ploxiln)
* #726 - add `.exe` extension to Windows binaries (thanks @ploxiln)
* #722 - `nsqadmin`: fix connected duration > `1hr`
### 0.3.7 - 2016-02-23
**Upgrading from 0.3.6**: Binaries contain no backwards incompatible changes.
This release has been built with Go 1.6.
Highlights include the various work done to reduce `nsqd` lock contention, significantly improving
the impact of high load on the `/stats` endpoint, addressing issues with timeouts and failures
in `nsqadmin` (#700, #701, #703, #709).
Thanks to @judwhite, `nsqd` and `nsqlookupd` now natively support being run as a Windows service
(#718). We're also now publishing official Windows releases.
`nsqd` will now `flock` its data directory on linux, preventing two `nsqd` from running
simultaneously pointed at the same path (#583).
On the bugfix side, the most noteworthy change is that `nsqd` will now correctly reset health state
on a successful backend write (#671).
Features:
* #700/#701/#703/#709 - `nsqd`: reduce lock contention (thanks @zachbadgett @absolute8511)
* #718 - `nsqd`/`nsqlookupd`: support running as a windows service (thanks @judwhite)
* #706 - `nsqd`: support enabling/disabling block profile via HTTP (thanks @absolute8511)
* #710 - `nsqd`: support `POST` `/debug/pprof/symbol` (thanks @absolute8511)
* #662 - `nsqadmin`: add flags for formatting statsd keys (thanks @kesutton)
* #583 - `nsqd`: `flock` `--data-path` on linux
* #663 - `nsqd`: optimize GUID generation (thanks @ploxiln)
Bugs:
* #672 - `nsqd`: fix max size accounting in `diskqueue` (thanks @judwhite)
* #671 - `nsqd`: reset health on successful backend write (thanks @judwhite)
* #615 - `nsqd`: prevent OOM when reading from `nsqlookupd` peer
* #664/#666 - dist.sh/Makefile cleanup (thanks @ploxiln)
### 0.3.6 - 2015-09-24
**Upgrading from 0.3.5**: Binaries contain no backwards incompatible changes.
We've adopted the [Contributor Covenant 1.2 Code of Conduct](CODE_OF_CONDUCT.md) (#593). Help us
keep NSQ open and inclusive by reading and following this document.
We closed a few longstanding issues related to `nsqadmin`, namely (#323, et al.) converting it to
an API and single-page app (so that it is _much_ easier to develop), displaying fine-grained errors
(#421, #657), and enabling support for `--tls-required` configurations (#396).
For `nsqd`, we added support for deferred publishing aka `DPUB` (#293), which allows a producer to
specify a duration of time to delay initial delivery of the message. We also addressed performance
issues relating to large numbers of topics/channels (#577) by removing some per-channel goroutines
in favor of a centralized, periodic, garbage collection approach.
In order to provide more flexibility when deploying NSQ in dynamically orchestrated topologies,
`nsqd` now supports the ability to configure `nsqlookupd` peers at runtime via HTTP (#601),
eliminating the need to restart the daemon.
As part of the large `nsqadmin` refactoring, we took the opportunity to cleanup the internals for
_all_ of the daemon's HTTP code paths (#601, #610, #612, #641) as well as improving the test suite
so that it doesn't leave around temporary files (#553).
Features:
* #593 - add code of conduct
* #323/#631/#632/#642/#421/#649/#650/#651/#652/#654 - `nsqadmin`: convert to API / single-page app
* #653 - `nsqadmin`: expand notification context
* #293 - `nsqd`: add deferred pub (`DPUB`)
* #577 - `nsqd`: drop per-channel queue workers in favor of centralized queue GC
* #584 - `nsqlookupd`: improve registration DB performance (thanks @xiaost)
* #601 - `nsqd`: HTTP endpoints to dynamically configure `nsqlookupd` peers
* #608 - `nsqd`: support for filtering `/stats` to topic/channel (thanks @chrusty)
* #601/#610/#612/#641 - improved HTTP internal routing / log HTTP requests
* #628 - `nsqd`: clarify help text for `--e2e-processing-latency-percentile`
* #640 - switch `--{consumer,producer}-opt` to `nsq.ConfigFlag`
Bugs:
* #656 - `nsqadmin`: update `statsd` prefix to `stats.counters`
* #421/#657 - `nsqadmin`: display upstream/partial errors
* #396 - `nsqdamin`/`nsqd`: support for `--tls-required`
* #558 - don't overwrite docker root FS
* #582 - `nsqd`: ignore benign EOF errors
* #587 - `nsqd`: GUID error handling / catch errors if GUID goes backwards (thanks @mpe)
* #586 - `nsqd`: fix valid range for `--worker-id`
* #550/#602/#617/#618/#619/#620/#622 - `nsqd`: fix benchmarks (thanks @Dieterbe)
* #553 - cleanup test dirs
* #600 - `nsqd`: enforce diskqueue min/max message size (thanks @twmb)
### 0.3.5 - 2015-04-26
**Upgrading from 0.3.3**: Binaries contain no backwards incompatible changes.
This is another quick bug fix release to address the broken `nsqadmin` binary in the distribution
(see #578).
### 0.3.4 - 2015-04-26
**WARNING**: please upgrade to `v0.3.5` to address the broken `nsqadmin` binary.
**Upgrading from 0.3.3**: Binaries contain no backwards incompatible changes.
This is a quick bug fix release to fix the outdated `go-nsq` dependency in `v0.3.3`
for the bundled utilities (see 6e8504e).
### 0.3.3 - 2015-04-26
**WARNING**: please upgrade to `v0.3.5` to address the outdated `go-nsq` dependency for the
bundled utilities and the broken `nsqadmin` binary.
**Upgrading from 0.3.2**: Binaries contain no backwards incompatible changes.
This release is primarily a bug fix release after cleaning up and reorganizing the codebase.
`nsqadmin` is now importable, which paves the way for completing #323. The bundled utilities
received a few feature additions and bug fixes (mostly from bug fixes on the `go-nsq` side).
Features:
* #569 - `nsqadmin`: re-org into importable package
* #562 - `nsq_to_{nsq,http}`: add `epsilon-greedy` mode (thanks @twmb)
* #547 - `nsqd`: adds `start_time` to `/stats` (thanks @ShawnSpooner)
* #544 - `nsq_to_http`: accept any `200` response as success (thanks @mikedewar)
* #548 - `nsq_to_http`: read entire request body (thanks @imgix)
* #552/#554/#555/#556/#561 - code cleanup and `/internal` package re-org (thanks @cespare)
Bugs:
* #573 - `nsqd`: don't persist metadata upon startup (thanks @xiaost)
* #560 - `nsqd`: do not print `EOF` error when client closes cleanly (thanks @twmb)
* #557 - `nsqd`: fix `--tls-required=tcp-https` with `--tls-client-auth-policy` (thanks @twmb)
* #545 - enable shell expansion in official Docker image (thanks @paddyforan)
NOTE: the bundled utilities are built against [`go-nsq` `v1.0.4`][go-nsq_104] and include all of
those features/fixes.
[go-nsq_104]: https://github.com/nsqio/go-nsq/releases/tag/v1.0.4
### 0.3.2 - 2015-02-08
**Upgrading from 0.3.1**: Binaries contain no backwards incompatible changes however as of this
release we've updated our official Docker images.
We now provide a single Docker image [`nsqio/nsq`](https://registry.hub.docker.com/r/nsqio/nsq/)
that includes *all* of the NSQ binaries. We did this for several reasons, primarily because the
tagged versions in the previous incarnation were broken (and did not actually pin to a version!).
The new image is an order of magnitude smaller, weighing in around 70mb.
In addition, the impetus for this quick release is to address a slew of reconnect related bug fixes
in the utility apps (`nsq_to_nsq`, `nsq_to_file`, etc.), for details see the [`go-nsq` `v1.0.3`
release notes](https://github.com/nsqio/go-nsq/releases/tag/v1.0.3).
Features:
* #534/#539/#540 - single Dockerfile approach (thanks @paddyforan)
Bugs:
* #529 - nsqadmin: fix more `#ephemeral` topic deletion issues
* #530 - nsqd: fix the provided sample config file (thanks @jnewmano)
* #538 - nsqd: fix orphaned ephemeral channels (thanks @adamsathailo)
### 0.3.1 - 2015-01-21
**Upgrading from 0.3.0**: No backwards incompatible changes.
This release contains minor bug fixes and feature additions.
There are a number of functionality improvements to the `nsq_stat` and `nsq_to_file` helper
applications (and general support for `#ephemeral` topics, broken in `0.2.30`).
Additionally, the TLS options continue to improve with support for setting `--tls-min-version` and
a work-around for a bug relating to `TLS_FALLBACK_SCSV` ([to be fixed in Go
1.5](https://go-review.googlesource.com/#/c/1776/)).
Features:
* #527 - nsq_stat: deprecate `--status-every` in favor of `--interval`
* #524 - nsq_stat: add `--count` option (thanks @nordicdyno)
* #518 - nsqd: set defaults for `--tls-min-version` and set TLS max version to 1.2
* #475/#513/#518 - nsqd: `--tls-required` can be disabled for HTTP / add `--tls-min-version`
(thanks @twmb)
* #496 - nsq_to_file: add `<PID>` to filename and rotation by size/interval (thanks @xiaost)
* #507 - nsq_stat: add rates (thanks @xiaost)
* #505 - nsqd: speed up failure path of `BytesToBase10` (thanks @iand)
Bugs:
* #522 - nsqadmin: fix `#ephemeral` topic deletion issues
* #509 - nsqd: fix `diskqueue` atomic rename on Windows (thanks @allgeek)
* #479 - nsqd: return `output_buffer_*` resolved settings in `IDENTIFY` response (thanks @tj)
### 0.3.0 - 2014-11-18
**Upgrading from 0.2.31**: No backwards incompatible changes.
This release includes a slew of bug fixes and few key feature additions.
The biggest functional change is that `nsqd` no longer decrements its `RDY` count for clients. This
means that client libraries no longer have to periodically re-send `RDY`. For some context, `nsqd`
already provided back-pressure due to the fact that a client must respond to messages before
receiving new ones. The decremented `RDY` count only made the implementation of the server and
client more complex without additional benefit. Now the `RDY` command can be treated as an "on/off"
switch. For details see #404 and the associated changes in nsqio/go-nsq#83 and nsqio/pynsq#98.
The second biggest change (and oft-requested feature!) is `#ephemeral` topics. Their behavior
mirrors that of channels. This feature is incredibly useful for situations where you're using
topics to "route" messages to consumers (like RPC) or when a backlog of messages is undesirable.
There are now scripts in the `bench` directory that automate the process of running a distributed
benchmark. This is a work-in-progress, but it already provides a closer-to-production setup and
therefore more accurate results. There's much work to do here!
A whole bunch of bugs were fixed - notably all were 3rd-party contributions! Thanks!
* #305 - `#ephemeral` topics
* #404/#459 - don't decr `RDY` / send `RDY` before `FIN`/`REQ`
* #472 - improve `nsqd` `diskqueue` sync strategies
* #488 - ability to filter topics by regex in `nsq_to_file` (thanks @lxfontes)
* #438 - distributed pub-sub benchmark scripts
* #448 - better `nsqd` `IOLoop` logging (thanks @rexposadas)
* #458 - switch to [gpm](https://github.com/pote/gpm) for builds
Bugs:
* #493 - ensure all `nsqd` `Notify()` goroutines have exited prior to shutdown (thanks @allgeek)
* #492 - ensure `diskqueue` syncs at end of benchmarks (thanks @Dieterbe)
* #490 - de-flake `TestPauseMetadata` (thanks @allgeek)
* #486 - require ports to be specified for daemons (thanks @jnewmano)
* #482 - better bash in `dist.sh` (thanks @losinggeneration)
* #480 - fix panic when `nsqadmin` checks stats for missing topic (thanks @jnewmano)
* #469 - fix panic when misbehaving client sends corrupt command (thanks @prio)
* #461 - fix panic when `nsqd` decodes corrupt message data (thanks @twmb)
* #454/#455 - fix 32-bit atomic ops in `nsq_to_nsq`/`nsq_to_http` (thanks @leshik)
* #451 - fix `go get` compatibility (thanks @adams-sarah)
### 0.2.31 - 2014-08-26
**Upgrading from 0.2.30**: No backwards incompatible changes.
This release includes a few key changes. First, we improved feedback and back-pressure when `nsqd`
writes to disk. Previously this was asynchronous and would result in clients not knowing that their
`PUB` had failed. Interestingly, this refactoring improved performance of `PUB` by 41%, by removing
the topic's goroutine responsible for message routing in favor of `N:N` Go channel communication.
For details see #437.
@paddyforan contributed official Dockerfiles that are now built automatically via Docker Hub.
Please begin to use (and improve these) as the various older images we had been maintaining will be
deprecated.
The utility apps deprecated the `--reader-opt` flag in favor of `--consumer-opt` and `nsq_to_nsq`
and `to_nsq` received a `--producer-opt` flag, for configuring details of the connection publishing
to `nsqd`. Additionally, it is now possible to configure client side TLS certificates via
`tls_cert` and `tls_key` opts.
As usual, we fixed a few minor bugs, see below for details.
New Features / Enhancements:
* #422/#437 - `nsqd`: diskqueue error feedback/backpressure (thanks @boyand)
* #412 - official Dockerfiles for `nsqd`, `nsqlookupd`, `nsqadmin` (thanks @paddyforan)
* #442 - utilities: add `--consumer-opt` alias for `--reader-opt` and
add `--producer-opt` to `nsq_to_nsq` (also support configuration
of `tls_cert` and `tls_key`)
* #448 - `nsqd`: improve IOLoop error messages (thanks @rexposadas)
Bugs:
* #440 - `nsqd`: fixed statsd GC stats reporting (thanks @jphines)
* #434/#435 - refactored/stabilized tests and logging
* #429 - `nsqd`: improve handling/documentation of `--worker-id` (thanks @bschwartz)
* #428 - `nsqd`: `IDENTIFY` should respond with materialized `msg_timeout` (thanks @visionmedia)
### 0.2.30 - 2014-07-28
**Upgrading from 0.2.29**: No backwards incompatible changes.
**IMPORTANT**: this is a quick bug-fix release to address a panic in `nsq_to_nsq` and
`nsq_to_http`, see #425.
New Features / Enhancements:
* #417 - `nsqadmin`/`nsqd`: expose TLS connection state
* #425 - `nsq_to_nsq`/`nsq_to_file`: display per-destination-address timings
Bugs:
* #425 - `nsq_to_nsq`/`nsq_to_file`: fix shared mutable state panic
### 0.2.29 - 2014-07-25
**Upgrading from 0.2.28**: No backwards incompatible changes.
This release includes a slew of new features and bug fixes, with contributions from 8
members of the community, thanks!
The most important new feature is authentication (the `AUTH` command for `nsqd`), added in #356.
When `nsqd` is configured with an `--auth-http-address` it will require clients to send the `AUTH`
command. The `AUTH` command body is opaque to `nsqd`, it simply passes it along to the configured
auth daemon which responds with well formed JSON, indicating which topics/channels and properties
on those entities are accessible to that client (rejecting the client if it accesses anything
prohibited). For more details, see [the spec](https://nsq.io/clients/tcp_protocol_spec.html) or [the
`nsqd` guide](https://nsq.io/components/nsqd.html#auth).
Additionally, we've improved performance in a few areas. First, we refactored in-flight handling in
`nsqd` to reduce garbage creation and improve baseline performance 6%. End-to-end processing
latency calculations are also significantly faster, thanks to improvements in the
[`perks`](https://github.com/bmizerany/perks/pulls/7) package.
HTTP response formats have been improved (removing the redundant response wrapper) and cleaning up
some of the endpoint namespaces. This change is backwards compatible. Clients wishing to move
towards the new response format can either use the new endpoint names or send the following header:
Accept: application/vnd.nsq version=1.0
Other changes including officially bumping the character limit for topic and channel names to 64
(thanks @svmehta), making the `REQ` timeout limit configurable in `nsqd` (thanks @AlphaB), and
compiling static asset dependencies into `nsqadmin` to simplify deployment (thanks @crossjam).
Finally, `to_nsq` was added to the suite of bundled apps. It takes a stdin stream and publishes to
`nsqd`, an extremely flexible solution (thanks @matryer)!
As for bugs, they're mostly minor, see the pull requests referenced in the section below for
details.
New Features / Enhancements:
* #304 - apps: added `to_nsq` for piping stdin to NSQ (thanks @matryer)
* #406 - `nsqadmin`: embed external static asset dependencies (thanks @crossjam)
* #389 - apps: report app name and version via `user_agent`
* #378/#390 - `nsqd`: improve in-flight message handling (6% faster, GC reduction)
* #356/#370/#386 - `nsqd`: introduce `AUTH`
* #358 - increase topic/channel name max length to 64 (thanks @svmehta)
* #357 - remove internal `go-nsq` dependencies (GC reduction)
* #330/#366 - version HTTP endpoints, simplify response format
* #352 - `nsqd`: make `REQ` timeout limit configurable (thanks @AlphaB)
* #340 - `nsqd`: bump perks dependency (E2E performance improvement, see 25086e4)
Bugs:
* #384 - `nsqd`: fix statsd GC time reporting
* #407 - `nsqd`: fix double `TOUCH` and use of client's configured msg timeout
* #392 - `nsqadmin`: fix HTTPS warning (thanks @juliangruber)
* #383 - `nsqlookupd`: fix race on last update timestamp
* #385 - `nsqd`: properly handle empty `FIN`
* #365 - `nsqd`: fix `IDENTIFY` `msg_timeout` response (thanks @visionmedia)
* #345 - `nsq_to_file`: set proper permissions on new directories (thanks @bschwartz)
* #338 - `nsqd`: fix windows diskqueue filenames (thanks @politician)
### 0.2.28 - 2014-04-28
**Upgrading from 0.2.27**: No backwards incompatible changes. We've deprecated the `short_id`
and `long_id` options in the `IDENTIFY` command in favor of `client_id` and `hostname`, which
more accurately reflect the data typically used.
This release includes a few important new features, in particular enhanced `nsqd`
TLS support thanks to a big contribution by @chrisroberts.
You can now *require* that clients negotiate TLS with `--tls-required` and you can configure a
client certificate policy via `--tls-client-auth-policy` (`require` or `require-verify`):
* `require` - the client must offer a certificate, otherwise rejected
* `require-verify` - the client must offer a valid certificate according to the default CA or
the chain specified by `--tls-root-ca-file`, otherwise rejected
This can be used as a form of client authentication.
Additionally, `nsqd` is now structured such that it is importable in other Go applications
via `github.com/nsqio/nsq/nsqd`, thanks to @kzvezdarov.
Finally, thanks to @paddyforan, `nsq_to_file` can now archive *multiple* topics or
optionally archive *all* discovered topics (by specifying no `--topic` params
and using `--lookupd-http-address`).
New Features / Enhancements:
* #334 - `nsq_to_file` can archive many topics (thanks @paddyforan)
* #327 - add `nsqd` TLS client certificate verification policy, ability
to require TLS, and HTTPS support (thanks @chrisroberts)
* #325 - make `nsqd` importable (`github.com/nsqio/nsq/nsqd`) (thanks @kzvezdarov)
* #321 - improve `IDENTIFY` options (replace `short_id` and `long_id` with
`client_id` and `hostname`)
* #319 - allow path separator in `nsq_to_file` filenames (thanks @jsocol)
* #324 - display memory depth and total depth in `nsq_stat`
Bug Fixes:
* nsqio/go-nsq#19 and nsqio/go-nsq#29 - fix deadlocks on `nsq.Reader` connection close/exit, this
impacts the utilities packaged with the NSQ binary
distribution such as `nsq_to_file`, `nsq_to_http`,
`nsq_to_nsq` and `nsq_tail`.
* #329 - use heartbeat interval for write deadline
* #321/#326 - improve benchmarking tests
* #315/#318 - fix test data races / flakiness
### 0.2.27 - 2014-02-17
**Upgrading from 0.2.26**: No backwards incompatible changes. We deprecated `--max-message-size`
in favor of `--max-msg-size` for consistency with the rest of the flag names.
IMPORTANT: this is another quick bug-fix release to address an issue in `nsqadmin` where templates
were incompatible with older versions of Go (pre-1.2).
* #306 - fix `nsqadmin` template compatibility (and formatting)
* #310 - fix `nsqadmin` behavior when E2E stats are disabled
* #309 - fix `nsqadmin` `INVALID_ERROR` on node page tombstone link
* #311/#312 - fix `nsqd` client metadata race condition and test flakiness
* #314 - fix `nsqd` test races (run w/ `-race` and `GOMAXPROCS=4`) deprecate `--max-message-size`
### 0.2.26 - 2014-02-06
**Upgrading from 0.2.25**: No backwards incompatible changes.
IMPORTANT: this is a quick bug-fix release to address a regression identified in `0.2.25` where
`statsd` prefixes were broken when using the default (or any) prefix that contained a `%s` for
automatic host replacement.
* #303 - fix `nsqd` `--statsd-prefix` when using `%s` host replacement
### 0.2.25 - 2014-02-05
**Upgrading from 0.2.24**: No backwards incompatible changes.
This release adds several commonly requested features.
First, thanks to [@elubow](https://twitter.com/elubow) you can now configure your clients to sample
the stream they're subscribed to. To read more about the details of the implementation see #286 and
the original discussion in #223. Eric also contributed an improvement to `nsq_tail` to add
the ability to tail the last `N` messages and exit.
We added config file support ([TOML](https://github.com/mojombo/toml/blob/master/README.md)) for
`nsqd`, `nsqlookupd`, and `nsqadmin` - providing even more deployment flexibility. Example configs
are in the `contrib` directory. Command line arguments override the equivalent option in the config
file.
We added the ability to pause a *topic* (it is already possible to pause individual *channels*).
This functionality stops all message flow from topic to channel for *all channels* of a topic,
queueing at the topic level. This enables all kinds of interesting possibilities like atomic
channel renames and trivial infrastructure wide operations.
Finally, we now compile the static assets used by `nsqadmin` into the binary, simplifying
deployment. This means that `--template-dir` is now deprecated and will be removed in a future
release and you can remove the templates you previously deployed and maintained.
New Features / Enhancements:
* #286 - add client `IDENTIFY` option to sample a % of messages
* #279 - add TOML config file support to `nsqd`, `nsqlookupd`, and `nsqadmin`
* #263 - add ability to pause a topic
* #291 - compile templates into `nsqadmin` binary
* #285/#288 - `nsq_tail` support for `-n #` to get recent # messages
* #287/#294 - display client `IDENTIFY` attributes in `nsqadmin` (sample rate, TLS, compression)
* #189/#296 - add client user agent to `nsqadmin``
* #297 - add `nsq_to_nsq` JSON message filtering options
### 0.2.24 - 2013-12-07
**Upgrading from 0.2.23**: No backwards incompatible changes. However, as you'll see below, quite a
few command line flags to the utility apps (`nsq_to_http`, `nsq_to_file`, `nsq_to_http`) were
deprecated and will be removed in the next release. Please use this release to transition over to
the new ones.
NOTE: we are now publishing additional binaries built against go1.2
The most prominent addition is the tracking of end-to-end message processing percentiles. This
measures the amount of time it's taking from `PUB` to `FIN` per topic/channel. The percentiles are
configurable and, because there is *some* overhead in collecting this data, it can be turned off
entirely. Please see [the section in the docs](https://nsq.io/components/nsqd.html) for
implementation details.
Additionally, the utility apps received comprehensive support for all configurable reader options
(including compression, which was previously missing). This necessitated a bit of command line flag
cleanup, as follows:
#### nsq_to_file
* deprecated `--gzip-compression` in favor of `--gzip-level`
* deprecated `--verbose` in favor of `--reader-opt=verbose`
#### nsq_to_http
* deprecated `--throttle-fraction` in favor of `--sample`
* deprecated `--http-timeout-ms` in favor of `--http-timeout` (which is a
*duration* flag)
* deprecated `--verbose` in favor of `--reader-opt=verbose`
* deprecated `--max-backoff-duration` in favor of
`--reader-opt=max_backoff_duration=X`
#### nsq_to_nsq
* deprecated `--verbose` in favor of `--reader-opt=verbose`
* deprecated `--max-backoff-duration` in favor of
`--reader-opt=max_backoff_duration=X`
New Features / Enhancements:
* #280 - add end-to-end message processing latency metrics
* #267 - comprehensive reader command line flags for utilities
### 0.2.23 - 2013-10-21
**Upgrading from 0.2.22**: No backwards incompatible changes.
We now use [godep](https://github.com/kr/godep) in order to achieve reproducible builds with pinned
dependencies. If you're on go1.1+ you can now just use `godep get github.com/nsqio/nsq/...`.
This release includes `nsqd` protocol compression feature negotiation.
[Snappy](https://code.google.com/p/snappy/) and [Deflate](http://en.wikipedia.org/wiki/DEFLATE) are
supported, clients can choose their preferred format.
`--statsd-prefix` can now be used to modify the prefix for the `statsd` keys generated by `nsqd`.
This is useful if you want to add datacenter prefixes or remove the default host prefix.
Finally, this release includes a "bug" fix that reduces CPU usage for `nsqd` with many clients by
choosing a more reasonable default for a timer used in client output buffering. For more details
see #236.
New Features / Enhancements:
* #266 - use godep for reproducible builds
* #229 - compression (Snappy/Deflate) feature negotiation
* #241 - binary support for HTTP /mput
* #269 - add --statsd-prefix flag
Bug Fixes:
* #278 - fix nsqd race for client subscription cleanup (thanks @simplereach)
* #277 - fix nsqadmin counter page
* #275 - stop accessing simplejson internals
* #274 - nsqd channel pause state lost during unclean restart (thanks @hailocab)
* #236 - reduce "idle" CPU usage by 90% with large # of clients
### 0.2.22 - 2013-08-26
**Upgrading from 0.2.21**: message timestamps are now officially nanoseconds. The protocol docs
always stated this however `nsqd` was actually sending seconds. This may cause some compatibility
issues for client libraries/clients that were taking advantage of this field.
This release also introduces support for TLS feature negotiation in `nsqd`. Clients can optionally
enable TLS by using the appropriate handshake via the `IDENTIFY` command. See #227.
Significant improvements were made to the HTTP publish endpoints and in flight message handling to
reduce GC pressure and eliminate memory abuse vectors. See #242, #239, and #245.
This release also includes a new utility `nsq_to_nsq` for performant, low-latency, copying of an NSQ
topic over the TCP protocol.
Finally, a whole suite of debug HTTP endpoints were added (and consolidated) under the
`/debug/pprof` namespace. See #238, #248, and #252. As a result `nsqd` now supports *direct*
profiling via Go's `pprof` tool, ie:
$ go tool pprof --web http://ip.address:4151/debug/pprof/heap
New Features / Enhancements:
* #227 - TLS feature negotiation
* #238/#248/#252 - support for more HTTP debug endpoints
* #256 - `nsqadmin` single node view (with GC/mem graphs)
* #255 - `nsq_to_nsq` utility for copying a topic over TCP
* #230 - `nsq_to_http` takes `--content-type` flag (thanks @michaelhood)
* #228 - `nsqadmin` displays tombstoned topics in the `/nodes` list
* #242/#239/#245 - reduced GC pressure for inflight and `/mput`
Bug Fixes:
* #260 - `tombstone_topic_producer` action in `nsqadmin` missing node info
* #244 - fix 64bit atomic alignment issues on 32bit platforms
* #251 - respect configured limits for HTTP publishing
* #247 - publish methods should not allow 0 length messages
* #231/#259 - persist `nsqd` metadata on topic/channel changes
* #237 - fix potential memory leaks with retained channel references
* #232 - message timestamps are now nano
* #228 - `nsqlookupd`/`nsqadmin` would display inactive nodes in `/nodes` list
* #216 - fix edge cases in `nsq_to_file` that caused empty files
### 0.2.21 - 2013-06-07
**Upgrading from 0.2.20**: there are no backward incompatible changes in this release.
This release introduces a significant new client feature as well as a slew of consistency and
recovery improvements to diskqueue.
First, we expanded the feature negotiation options for clients. There are many cases where you want
different output buffering semantics from `nsqd` to your client. You can now control both
output buffer size and the output buffer timeout via new fields in the `IDENTIFY` command. You can
even disable output buffering if low latency is a priority.
You can now specify a duration between fsyncs via `--sync-timeout`. This is a far better way to
manage when the process fsyncs messages to disk (vs the existing `--sync-every` which is based on #
of messages). `--sync-every` is now considered a deprecated option and will be removed in a future
release.
Finally, `0.2.20` introduced a significant regression in #176 where a topic would not write messages
to its channels. It is recommended that all users running `0.2.20` upgrade to this release. For
additional information see #217.
New Features / Enhancements:
* #214 - add --sync-timeout for time based fsync, improve when diskqueue syncs
* #196 - client configurable output buffering
* #190 - nsq_tail generates a random #ephemeral channel
Bug Fixes:
* #218/#220 - expose --statsd-interval for nsqadmin to handle non 60s statsd intervals
* #217 - fix new topic channel creation regression from #176 (thanks @elubow)
* #212 - dont use port in nsqadmin cookies
* #214 - dont open diskqueue writeFile with O_APPEND
* #203/#211 - diskqueue depth accounting consistency
* #207 - failure to write a heartbeat is fatal / reduce error log noise
* #206 - use broadcast address for statsd prefix
* #205 - cleanup example utils exit
### 0.2.20 - 2013-05-13
**Upgrading from 0.2.19**: there are no backward incompatible changes in this release.
This release adds a couple of convenient features (such as adding the ability to empty a *topic*)
and continues our work to reduce garbage produced at runtime to relieve GC pressure in the Go
runtime.
`nsqd` now has two new flags to control the max value clients can use to set their heartbeat
interval as well as adjust a clients maximum RDY count. This is all set/communicated via `IDENTIFY`.
`nsqadmin` now displays `nsqd` -> `nsqlookupd` connections in the "nodes" view. This is useful for
visualizing how the topology is connected as well as situations where `--broadcast-address` is being
used incorrectly.
`nsq_to_http` now has a "host pool" mode where upstream state will be adjusted based on
successful/failed requests and for failures, upstreams will be exponentially backed off. This is an
incredibly useful routing mode.
As for bugs, we fixed an issue where "fatal" client errors were not actually being treated as fatal.
Under certain conditions deleting a topic would not clean up all of its files on disk. There was a
reported issue where the `--data-path` was not writable by the process and this was only discovered
after message flow began. We added a writability check at startup to improve feedback. Finally.
`deferred_count` was being sent as a counter value to statsd, it should be a gauge.
New Features / Enhancements:
* #197 - nsqadmin nodes list improvements (show nsqd -> lookupd conns)
* #192 - add golang runtime version to daemon version output
* #183 - ability to empty a topic
* #176 - optimizations to reduce garbage, copying, locking
* #184 - add table headers to nsqadmin channel view (thanks @elubow)
* #174/#186 - nsq_to_http hostpool mode and backoff control
* #173/#187 - nsq_stat utility for command line introspection
* #175 - add nsqd --max-rdy-count configuration option
* #178 - add nsqd --max-heartbeat-interval configuration option
Bug Fixes:
* #198 - fix fatal errors not actually being fatal
* #195 - fix delete topic does not delete all diskqueue files
* #193 - fix data race in channel requeue
* #185 - ensure that --data-path is writable on startup
* #182 - fix topic deletion ordering to prevent race conditions with lookupd/diskqueue
* #179 - deferred_count as gauge for statsd
* #173/#188/#191 - fix nsqadmin counter template error; fix nsqadmin displaying negative rates
### 0.2.19 - 2013-04-11
**Upgrading from 0.2.18**: there are no backward incompatible changes in this release.
This release is a small release that introduces one major client side feature and resolves one
critical bug.
`nsqd` clients can now configure their own heartbeat interval. This is important because as of
`0.2.18` *all* clients (including producers) received heartbeats by default. In certain cases
receiving a heartbeat complicated "simple" clients that just wanted to produce messages and not
handle asynchronous responses. This gives flexibility for the client to decide how it would like
behave.
A critical bug was discovered where emptying a channel would leave client in-flight state
inconsistent (it would not zero) which limited deliverability of messages to those clients.
New Features / Enhancements:
* #167 - 'go get' compatibility
* #158 - allow nsqd clients to configure (or disable) heartbeats
Bug Fixes:
* #171 - fix race conditions identified testing against go 1.1 (scheduler improvements)
* #160 - empty channel left in-flight count inconsistent (thanks @dmarkham)
### 0.2.18 - 2013-02-28
**Upgrading from 0.2.17**: all V2 clients of nsqd now receive heartbeats (previously only clients
that subscribed would receive heartbeats, excluding TCP *producers*).
**Upgrading from 0.2.16**: follow the notes in the 0.2.17 changelog for upgrading from 0.2.16.
Beyond the important note above regarding heartbeats this release includes `nsq_tail`, an extremely
useful utility application that can be used to introspect a topic on the command line. If statsd is
enabled (and graphite in `nsqadmin`) we added the ability to retrieve rates for display in
`nsqadmin`.
We resolved a few critical issues with data consistency in `nsqlookupd` when channels and topics are
deleted. First, deleting a topic would cause that producer to disappear from `nsqlookupd` for all
topics. Second, deleting a channel would cause that producer to disappear from the topic list in
`nsqlookupd`.
New Features / Enhancements:
* #131 - all V2 nsqd clients get heartbeats
* #154 - nsq_tail example reader
* #143 - display message rates in nsqadmin
Bug Fixes:
* #148 - store tombstone data per registration in nsqlookupd
* #153 - fix large graph formulas in nsqadmin
* #150/#151 - fix topics disappearing from nsqlookupd when channels are deleted
### 0.2.17 - 2013-02-07
**Upgrading from 0.2.16**: IDENTIFY and SUB now return success responses (they previously only
responded to errors). The official Go and Python libraries are forwards/backwards compatible with
this change however 3rd party client libraries may not be.
**Upgrading from 0.2.15**: in #132 deprecations in SUB were removed as well as support for the old,
line oriented, `nsqd` metadata file format. For these reasons you should upgrade to `0.2.16` first.
New Features / Enhancements:
* #119 - add TOUCH command to nsqd
* #142 - add --broadcast-address flag to nsqd/nsqadmin (thanks @dustismo)
* #135 - atomic MPUB
* #133 - improved protocol fatal error handling and responses; IDENTIFY/SUB success responses
* #118 - switch nsqadmin actions to POST and require confirmation
* #117/#147 - nsqadmin action POST notifications
* #122 - configurable msg size limits
* #132 - deprecate identify in SUB and old nsqd metadata file format
Bug Fixes:
* #144 - empty channel should clear inflight/deferred messages
* #140 - fix MPUB protocol documentation
* #139 - fix nsqadmin handling of legacy statsd prefixes for graphs
* #138/#145 - fix nsqadmin action redirect handling
* #134 - nsqd to nsqlookupd registration fixes
* #129 - nsq_to_file gzip file versioning
* #106 - nsqlookupd topic producer tombstones
* #100 - sane handling of diskqueue read errors
* #123/#125 - fix notify related exit deadlock
### 0.2.16 - 2013-01-07
**Upgrading from 0.2.15**: there are no backward incompatible changes in this release.
However, this release introduces the `IDENTIFY` command (which supersedes sending
metadata along with `SUB`) for clients of `nsqd`. The old functionality will be
removed in a future release.
* #114 persist paused channels through restart
* #121 fix typo preventing compile of bench_reader (thanks @datastream)
* #120 fix nsqd crash when empty command is sent (thanks @michaelhood)
* #115 nsq_to_file --filename-format --datetime-format parameter and fix
* #101 fix topic/channel delete operations ordering
* #98 nsqadmin fixes when not using lookupd
* #90/#108 performance optimizations / IDENTIFY protocol support in nsqd. For
a single consumer of small messages (< 4k) increases throughput ~400% and
reduces # of allocations ~30%.
* #105 strftime compatible datetime format
* #103 nsq_to_http handler logging
* #102 compatibility with Go tip
* #99 nsq_to_file --gzip flag
* #95 proxy graphite requests through nsqadmin
* #93 fix nqd API response for no topics
* #92 graph rendering options
* #86 nsq_to_http Content-Length headers
* #89 gopkg doc updates
* #88 move pynsq to it's own repo
* #81/#87 reader improvements / introduced MPUB. Fix bug for mem-queue-size < 10
* #76 statsd/graphite support
* #75 administrative ability to create topics and channels
### 0.2.15 - 2012-10-25
* #84 fix lookupd hanging on to ephemeral channels w/ no producers
* #82 add /counter page to nsqadmin
* #80 message size benchmark
* #78 send Content-Length for nsq_to_http requests
* #57/#83 documentation updates
### 0.2.14 - 2012-10-19
* #77 ability to pause a channel (includes bugfix for message pump/diskqueue)
* #74 propagate all topic changes to lookupd
* #65 create binary releases
### 0.2.13 - 2012-10-15
* #70 deadlined nsq_to_http outbound requests
* #69/#72 improved nsq_to_file sync strategy
* #58 diskqueue metadata bug and refactoring
### 0.2.12 - 2012-10-10
* #63 consolidated protocol V1 into V2 and fixed PUB bug
* #61 added a makefile for simpler building
* #55 allow topic/channel names with `.`
* combined versions for all binaries
### 0.2.7 - 0.2.11
* Initial public release.
## go-nsq Client Library
* #264 moved **go-nsq** to its own [repository](https://github.com/nsqio/go-nsq)
## pynsq Python Client Library
* #88 moved **pynsq** to its own [repository](https://github.com/nsqio/pynsq)
================================================
FILE: Dockerfile
================================================
FROM golang:alpine AS build
RUN apk update && apk add make gcc musl-dev
RUN mkdir -p /go/src/github.com/nsqio/nsq
COPY . /go/src/github.com/nsqio/nsq
WORKDIR /go/src/github.com/nsqio/nsq
RUN CGO_ENABLED=0 make BLDDIR=/tmp/nsq PREFIX=/opt/nsq BLDFLAGS='-ldflags="-s -w"' install
FROM alpine:latest
EXPOSE 4150 4151 4160 4161 4170 4171
RUN mkdir -p /data
WORKDIR /data
# Optional volumes (explicitly configure with "docker run -v ...")
# /data - used by nsqd for persistent storage across restarts
# /etc/ssl/certs - for SSL Root CA certificates from host
COPY --from=build /opt/nsq/bin/ /usr/local/bin/
RUN ln -s /usr/local/bin/*nsq* / \
&& ln -s /usr/local/bin/*nsq* /bin/
================================================
FILE: LICENSE
================================================
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in
all copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
THE SOFTWARE.
================================================
FILE: Makefile
================================================
PREFIX=/usr/local
BINDIR=${PREFIX}/bin
DESTDIR=
BLDDIR=build
BLDFLAGS=
EXT=
ifeq (${GOOS},windows)
EXT=.exe
endif
APPS = nsqd nsqlookupd nsqadmin nsq_to_nsq nsq_to_file nsq_to_http nsq_tail nsq_stat to_nsq
all: $(APPS)
$(BLDDIR)/nsqd: $(wildcard apps/nsqd/*.go nsqd/*.go nsq/*.go internal/*/*.go)
$(BLDDIR)/nsqlookupd: $(wildcard apps/nsqlookupd/*.go nsqlookupd/*.go nsq/*.go internal/*/*.go)
$(BLDDIR)/nsqadmin: $(wildcard apps/nsqadmin/*.go nsqadmin/*.go nsqadmin/templates/*.go internal/*/*.go)
$(BLDDIR)/nsq_to_nsq: $(wildcard apps/nsq_to_nsq/*.go nsq/*.go internal/*/*.go)
$(BLDDIR)/nsq_to_file: $(wildcard apps/nsq_to_file/*.go nsq/*.go internal/*/*.go)
$(BLDDIR)/nsq_to_http: $(wildcard apps/nsq_to_http/*.go nsq/*.go internal/*/*.go)
$(BLDDIR)/nsq_tail: $(wildcard apps/nsq_tail/*.go nsq/*.go internal/*/*.go)
$(BLDDIR)/nsq_stat: $(wildcard apps/nsq_stat/*.go internal/*/*.go)
$(BLDDIR)/to_nsq: $(wildcard apps/to_nsq/*.go internal/*/*.go)
$(BLDDIR)/%:
@mkdir -p $(dir $@)
go build ${BLDFLAGS} -o $@ ./apps/$*
$(APPS): %: $(BLDDIR)/%
clean:
rm -fr $(BLDDIR)
.PHONY: install clean all
.PHONY: $(APPS)
install: $(APPS)
install -m 755 -d ${DESTDIR}${BINDIR}
for APP in $^ ; do install -m 755 ${BLDDIR}/$$APP ${DESTDIR}${BINDIR}/$$APP${EXT} ; done
================================================
FILE: README.md
================================================
<p align="center">
<img align="left" width="175" src="https://nsq.io/static/img/nsq_blue.png">
<ul>
<li><strong>Source</strong>: https://github.com/nsqio/nsq
<li><strong>Issues</strong>: https://github.com/nsqio/nsq/issues
<li><strong>Mailing List</strong>: <a href="https://groups.google.com/d/forum/nsq-users">nsq-users@googlegroups.com</a>
<li><strong>IRC</strong>: #nsq on freenode
<li><strong>Docs</strong>: https://nsq.io
<li><strong>Twitter</strong>: <a href="https://twitter.com/nsqio">@nsqio</a>
</ul>
</p>
[](https://github.com/nsqio/nsq/actions) [](https://github.com/nsqio/nsq/releases/latest) [](https://coveralls.io/github/nsqio/nsq?branch=master)
**NSQ** is a realtime distributed messaging platform designed to operate at scale, handling
billions of messages per day.
It promotes *distributed* and *decentralized* topologies without single points of failure,
enabling fault tolerance and high availability coupled with a reliable message delivery
guarantee. See [features & guarantees][features_guarantees].
Operationally, **NSQ** is easy to configure and deploy (all parameters are specified on the command
line and compiled binaries have no runtime dependencies). For maximum flexibility, it is agnostic to
data format (messages can be JSON, MsgPack, Protocol Buffers, or anything else). Official Go and
Python libraries are available out of the box (as well as many other [client
libraries][client_libraries]), and if you're interested in building your own, there's a [protocol
spec][protocol].
We publish [binary releases][installing] for Linux, Darwin, FreeBSD and Windows, as well as an official [Docker image][docker_deployment].
NOTE: master is our *development* branch and may not be stable at all times.
## In Production
<a href="https://bitly.com/"><img src="https://nsq.io/static/img/bitly_logo.png" width="84" align="middle"/></a>
<a href="https://www.life360.com/"><img src="https://nsq.io/static/img/life360_logo.png" width="100" align="middle"/></a>
<a href="https://www.simplereach.com/"><img src="https://nsq.io/static/img/simplereach_logo.png" width="136" align="middle"/></a>
<a href="https://moz.com/"><img src="https://nsq.io/static/img/moz_logo.png" width="108" align="middle"/></a>
<a href="https://segment.com/"><img src="https://nsq.io/static/img/segment_logo.png" width="70" align="middle"/></a>
<a href="https://eventful.com/events"><img src="https://nsq.io/static/img/eventful_logo.png" width="95" align="middle"/></a><br/>
<a href="https://www.energyhub.com/"><img src="https://nsq.io/static/img/energyhub_logo.png" width="99" align="middle"/></a>
<a href="https://project-fifo.net/"><img src="https://nsq.io/static/img/project_fifo.png" width="97" align="middle"/></a>
<a href="https://trendrr.com/"><img src="https://nsq.io/static/img/trendrr_logo.png" width="97" align="middle"/></a>
<a href="https://reonomy.com/"><img src="https://nsq.io/static/img/reonomy_logo.png" width="100" align="middle"/></a>
<a href="https://hw-ops.com/"><img src="https://nsq.io/static/img/heavy_water.png" width="50" align="middle"/></a>
<a href="https://www.getlytics.com/"><img src="https://nsq.io/static/img/lytics.png" width="100" align="middle"/></a><br/>
<a href="https://mediaforge.com/"><img src="https://nsq.io/static/img/rakuten.png" width="100" align="middle"/></a>
<a href="https://wistia.com/"><img src="https://nsq.io/static/img/wistia_logo.png" width="140" align="middle"/></a>
<a href="https://stripe.com/"><img src="https://nsq.io/static/img/stripe_logo.png" width="96" align="middle"/></a>
<a href="https://www.shipwire.com/"><img src="https://nsq.io/static/img/shipwire_logo.png" width="97" align="middle"/></a>
<a href="https://digg.com/"><img src="https://nsq.io/static/img/digg_logo.png" width="97" align="middle"/></a>
<a href="https://www.scalabull.com/"><img src="https://nsq.io/static/img/scalabull_logo.png" width="97" align="middle"/></a><br/>
<a href="https://www.soundest.com/"><img src="https://nsq.io/static/img/soundest_logo.png" width="96" align="middle"/></a>
<a href="https://www.docker.com/"><img src="https://nsq.io/static/img/docker_logo.png" width="100" align="middle"/></a>
<a href="https://www.getweave.com/"><img src="https://nsq.io/static/img/weave_logo.png" width="94" align="middle"/></a>
<a href="https://www.augury.com/"><img src="https://nsq.io/static/img/augury_logo.png" width="97" align="middle"/></a>
<a href="https://www.buzzfeed.com/"><img src="https://nsq.io/static/img/buzzfeed_logo.png" width="97" align="middle"/></a>
<a href="https://eztable.com/"><img src="https://nsq.io/static/img/eztable_logo.png" width="97" align="middle"/></a><br/>
<a href="https://www.dotabuff.com/"><img src="https://nsq.io/static/img/dotabuff_logo.png" width="97" align="middle"/></a>
<a href="https://www.fastly.com/"><img src="https://nsq.io/static/img/fastly_logo.png" width="97" align="middle"/></a>
<a href="https://talky.io/"><img src="https://nsq.io/static/img/talky_logo.png" width="97" align="middle"/></a>
<a href="https://groupme.com/"><img src="https://nsq.io/static/img/groupme_logo.png" width="97" align="middle"/></a>
<a href="https://wiredcraft.com/"><img src="https://nsq.io/static/img/wiredcraft_logo.jpg" width="97" align="middle"/></a>
<a href="https://sproutsocial.com/"><img src="https://nsq.io/static/img/sproutsocial_logo.png" width="90" align="middle"/></a><br/>
<a href="https://fandom.wikia.com/"><img src="https://nsq.io/static/img/fandom_logo.svg" width="100" align="middle"/></a>
<a href="https://gitee.com/"><img src="https://nsq.io/static/img/gitee_logo.svg" width="140" align="middle"/></a>
<a href="https://bytedance.com/"><img src="https://nsq.io/static/img/bytedance_logo.png" width="140" align="middle"/></a>
<a href="https://www.tokopedia.com/"><img src="https://nsq.io/static/img/tokopedia_logo.svg" width="145" align="middle"/></a><br/>
<a href="https://www.kuafood.com/"><img src="https://nsq.io/static/img/kuafood_logo.png" width="145" align="middle"/></a><br/>
## Code of Conduct
Help us keep NSQ open and inclusive. Please read and follow our [Code of Conduct](CODE_OF_CONDUCT.md).
## Authors
NSQ was designed and developed by Matt Reiferson ([@imsnakes][snakes_twitter]) and Jehiah Czebotar
([@jehiah][jehiah_twitter]) but wouldn't have been possible without the support of [Bitly][bitly],
maintainers ([Pierce Lopez][pierce_github]), and all our [contributors][contributors].
Logo created by Wolasi Konu ([@kisalow][wolasi_twitter]).
[protocol]: https://nsq.io/clients/tcp_protocol_spec.html
[installing]: https://nsq.io/deployment/installing.html
[docker_deployment]: https://nsq.io/deployment/docker.html
[snakes_twitter]: https://twitter.com/imsnakes
[jehiah_twitter]: https://twitter.com/jehiah
[bitly]: https://bitly.com
[features_guarantees]: https://nsq.io/overview/features_and_guarantees.html
[contributors]: https://github.com/nsqio/nsq/graphs/contributors
[client_libraries]: https://nsq.io/clients/client_libraries.html
[wolasi_twitter]: https://twitter.com/kisalow
[pierce_github]: https://github.com/ploxiln
================================================
FILE: apps/nsq_stat/nsq_stat.go
================================================
// This is a utility application that polls /stats for all the producers
// of the specified topic/channel and displays aggregate stats
package main
import (
"errors"
"flag"
"fmt"
"log"
"os"
"os/signal"
"strconv"
"strings"
"syscall"
"time"
"github.com/nsqio/nsq/internal/app"
"github.com/nsqio/nsq/internal/clusterinfo"
"github.com/nsqio/nsq/internal/http_api"
"github.com/nsqio/nsq/internal/version"
)
var (
showVersion = flag.Bool("version", false, "print version")
topic = flag.String("topic", "", "NSQ topic")
channel = flag.String("channel", "", "NSQ channel")
interval = flag.Duration("interval", 2*time.Second, "duration of time between polling/printing output")
httpConnectTimeout = flag.Duration("http-client-connect-timeout", 2*time.Second, "timeout for HTTP connect")
httpRequestTimeout = flag.Duration("http-client-request-timeout", 5*time.Second, "timeout for HTTP request")
countNum = numValue{}
nsqdHTTPAddrs = app.StringArray{}
lookupdHTTPAddrs = app.StringArray{}
)
type numValue struct {
isSet bool
value int
}
func (nv *numValue) String() string { return "N" }
func (nv *numValue) Set(s string) error {
value, err := strconv.ParseInt(s, 10, 32)
if err != nil {
return err
}
nv.value = int(value)
nv.isSet = true
return nil
}
func init() {
flag.Var(&nsqdHTTPAddrs, "nsqd-http-address", "nsqd HTTP address (may be given multiple times)")
flag.Var(&lookupdHTTPAddrs, "lookupd-http-address", "lookupd HTTP address (may be given multiple times)")
flag.Var(&countNum, "count", "number of reports")
}
func statLoop(interval time.Duration, connectTimeout time.Duration, requestTimeout time.Duration,
topic string, channel string, nsqdTCPAddrs []string, lookupdHTTPAddrs []string) {
ci := clusterinfo.New(nil, http_api.NewClient(nil, connectTimeout, requestTimeout))
var o *clusterinfo.ChannelStats
for i := 0; !countNum.isSet || countNum.value >= i; i++ {
var producers clusterinfo.Producers
var err error
if len(lookupdHTTPAddrs) != 0 {
producers, err = ci.GetLookupdTopicProducers(topic, lookupdHTTPAddrs)
} else {
producers, err = ci.GetNSQDTopicProducers(topic, nsqdHTTPAddrs)
}
if err != nil {
log.Fatalf("ERROR: failed to get topic producers - %s", err)
}
_, channelStats, err := ci.GetNSQDStats(producers, topic, channel, false)
if err != nil {
log.Fatalf("ERROR: failed to get nsqd stats - %s", err)
}
c, ok := channelStats[channel]
if !ok {
log.Fatalf("ERROR: failed to find channel(%s) in stats metadata for topic(%s)", channel, topic)
}
if i%25 == 0 {
fmt.Printf("%s+%s+%s\n",
"------rate------",
"----------------depth----------------",
"--------------metadata---------------")
fmt.Printf("%7s %7s | %7s %7s %7s %5s %5s | %7s %7s %12s %7s\n",
"ingress", "egress",
"total", "mem", "disk", "inflt",
"def", "req", "t-o", "msgs", "clients")
}
if o == nil {
o = c
time.Sleep(interval)
continue
}
// TODO: paused
fmt.Printf("%7d %7d | %7d %7d %7d %5d %5d | %7d %7d %12d %7d\n",
int64(float64(c.MessageCount-o.MessageCount)/interval.Seconds()),
int64(float64(c.MessageCount-o.MessageCount-(c.Depth-o.Depth))/interval.Seconds()),
c.Depth,
c.MemoryDepth,
c.BackendDepth,
c.InFlightCount,
c.DeferredCount,
c.RequeueCount,
c.TimeoutCount,
c.MessageCount,
c.ClientCount)
o = c
time.Sleep(interval)
}
os.Exit(0)
}
func checkAddrs(addrs []string) error {
for _, a := range addrs {
if strings.HasPrefix(a, "http") {
return errors.New("address should not contain scheme")
}
}
return nil
}
func main() {
flag.Parse()
if *showVersion {
fmt.Printf("nsq_stat v%s\n", version.Binary)
return
}
if *topic == "" || *channel == "" {
log.Fatal("--topic and --channel are required")
}
intvl := *interval
if int64(intvl) <= 0 {
log.Fatal("--interval should be positive")
}
connectTimeout := *httpConnectTimeout
if int64(connectTimeout) <= 0 {
log.Fatal("--http-client-connect-timeout should be positive")
}
requestTimeout := *httpRequestTimeout
if int64(requestTimeout) <= 0 {
log.Fatal("--http-client-request-timeout should be positive")
}
if countNum.isSet && countNum.value <= 0 {
log.Fatal("--count should be positive")
}
if len(nsqdHTTPAddrs) == 0 && len(lookupdHTTPAddrs) == 0 {
log.Fatal("--nsqd-http-address or --lookupd-http-address required")
}
if len(nsqdHTTPAddrs) > 0 && len(lookupdHTTPAddrs) > 0 {
log.Fatal("use --nsqd-http-address or --lookupd-http-address not both")
}
if err := checkAddrs(nsqdHTTPAddrs); err != nil {
log.Fatalf("--nsqd-http-address error - %s", err)
}
if err := checkAddrs(lookupdHTTPAddrs); err != nil {
log.Fatalf("--lookupd-http-address error - %s", err)
}
termChan := make(chan os.Signal, 1)
signal.Notify(termChan, syscall.SIGHUP, syscall.SIGINT, syscall.SIGTERM)
go statLoop(intvl, connectTimeout, requestTimeout, *topic, *channel, nsqdHTTPAddrs, lookupdHTTPAddrs)
<-termChan
}
================================================
FILE: apps/nsq_tail/nsq_tail.go
================================================
package main
import (
"flag"
"fmt"
"log"
"math/rand"
"os"
"os/signal"
"syscall"
"time"
"github.com/nsqio/go-nsq"
"github.com/nsqio/nsq/internal/app"
"github.com/nsqio/nsq/internal/version"
)
var (
showVersion = flag.Bool("version", false, "print version string")
channel = flag.String("channel", "", "NSQ channel")
maxInFlight = flag.Int("max-in-flight", 200, "max number of messages to allow in flight")
totalMessages = flag.Int("n", 0, "total messages to show (will wait if starved)")
printTopic = flag.Bool("print-topic", false, "print topic name where message was received")
nsqdTCPAddrs = app.StringArray{}
lookupdHTTPAddrs = app.StringArray{}
topics = app.StringArray{}
)
func init() {
flag.Var(&nsqdTCPAddrs, "nsqd-tcp-address", "nsqd TCP address (may be given multiple times)")
flag.Var(&lookupdHTTPAddrs, "lookupd-http-address", "lookupd HTTP address (may be given multiple times)")
flag.Var(&topics, "topic", "NSQ topic (may be given multiple times)")
}
type TailHandler struct {
topicName string
totalMessages int
messagesShown int
}
func (th *TailHandler) HandleMessage(m *nsq.Message) error {
th.messagesShown++
if *printTopic {
_, err := os.Stdout.WriteString(th.topicName)
if err != nil {
log.Fatalf("ERROR: failed to write to os.Stdout - %s", err)
}
_, err = os.Stdout.WriteString(" | ")
if err != nil {
log.Fatalf("ERROR: failed to write to os.Stdout - %s", err)
}
}
_, err := os.Stdout.Write(m.Body)
if err != nil {
log.Fatalf("ERROR: failed to write to os.Stdout - %s", err)
}
_, err = os.Stdout.WriteString("\n")
if err != nil {
log.Fatalf("ERROR: failed to write to os.Stdout - %s", err)
}
if th.totalMessages > 0 && th.messagesShown >= th.totalMessages {
os.Exit(0)
}
return nil
}
func main() {
cfg := nsq.NewConfig()
flag.Var(&nsq.ConfigFlag{cfg}, "consumer-opt", "option to passthrough to nsq.Consumer (may be given multiple times, http://godoc.org/github.com/nsqio/go-nsq#Config)")
flag.Parse()
if *showVersion {
fmt.Printf("nsq_tail v%s\n", version.Binary)
return
}
if *channel == "" {
rand.Seed(time.Now().UnixNano())
*channel = fmt.Sprintf("tail%06d#ephemeral", rand.Int()%999999)
}
if len(nsqdTCPAddrs) == 0 && len(lookupdHTTPAddrs) == 0 {
log.Fatal("--nsqd-tcp-address or --lookupd-http-address required")
}
if len(nsqdTCPAddrs) > 0 && len(lookupdHTTPAddrs) > 0 {
log.Fatal("use --nsqd-tcp-address or --lookupd-http-address not both")
}
if len(topics) == 0 {
log.Fatal("--topic required")
}
sigChan := make(chan os.Signal, 1)
signal.Notify(sigChan, syscall.SIGINT, syscall.SIGTERM)
// Don't ask for more messages than we want
if *totalMessages > 0 && *totalMessages < *maxInFlight {
*maxInFlight = *totalMessages
}
cfg.UserAgent = fmt.Sprintf("nsq_tail/%s go-nsq/%s", version.Binary, nsq.VERSION)
cfg.MaxInFlight = *maxInFlight
consumers := []*nsq.Consumer{}
for i := 0; i < len(topics); i++ {
log.Printf("Adding consumer for topic: %s\n", topics[i])
consumer, err := nsq.NewConsumer(topics[i], *channel, cfg)
if err != nil {
log.Fatal(err)
}
consumer.AddHandler(&TailHandler{topicName: topics[i], totalMessages: *totalMessages})
err = consumer.ConnectToNSQDs(nsqdTCPAddrs)
if err != nil {
log.Fatal(err)
}
err = consumer.ConnectToNSQLookupds(lookupdHTTPAddrs)
if err != nil {
log.Fatal(err)
}
consumers = append(consumers, consumer)
}
<-sigChan
for _, consumer := range consumers {
consumer.Stop()
}
for _, consumer := range consumers {
<-consumer.StopChan
}
}
================================================
FILE: apps/nsq_to_file/file_logger.go
================================================
package main
import (
"compress/gzip"
"errors"
"fmt"
"io"
"os"
"path"
"path/filepath"
"strings"
"time"
"github.com/nsqio/go-nsq"
"github.com/nsqio/nsq/internal/lg"
)
type FileLogger struct {
logf lg.AppLogFunc
opts *Options
topic string
consumer *nsq.Consumer
out *os.File
writer io.Writer
gzipWriter *gzip.Writer
logChan chan *nsq.Message
filenameFormat string
termChan chan bool
hupChan chan bool
// for rotation
filename string
openTime time.Time
filesize int64
rev uint
}
func NewFileLogger(logf lg.AppLogFunc, opts *Options, topic string, cfg *nsq.Config) (*FileLogger, error) {
computedFilenameFormat, err := computeFilenameFormat(opts, topic)
if err != nil {
return nil, err
}
consumer, err := nsq.NewConsumer(topic, opts.Channel, cfg)
if err != nil {
return nil, err
}
f := &FileLogger{
logf: logf,
opts: opts,
topic: topic,
consumer: consumer,
logChan: make(chan *nsq.Message, 1),
filenameFormat: computedFilenameFormat,
termChan: make(chan bool),
hupChan: make(chan bool),
}
consumer.AddHandler(f)
err = consumer.ConnectToNSQDs(opts.NSQDTCPAddrs)
if err != nil {
return nil, err
}
err = consumer.ConnectToNSQLookupds(opts.NSQLookupdHTTPAddrs)
if err != nil {
return nil, err
}
return f, nil
}
func (f *FileLogger) HandleMessage(m *nsq.Message) error {
m.DisableAutoResponse()
f.logChan <- m
return nil
}
func (f *FileLogger) router() {
pos := 0
output := make([]*nsq.Message, f.opts.MaxInFlight)
sync := false
ticker := time.NewTicker(f.opts.SyncInterval)
closeFile := false
exit := false
for {
select {
case <-f.consumer.StopChan:
sync = true
closeFile = true
exit = true
case <-f.termChan:
ticker.Stop()
f.consumer.Stop()
sync = true
case <-f.hupChan:
sync = true
closeFile = true
case <-ticker.C:
if f.needsRotation() {
if f.opts.SkipEmptyFiles {
closeFile = true
} else {
f.updateFile()
}
}
sync = true
case m := <-f.logChan:
if f.needsRotation() {
f.updateFile()
sync = true
}
_, err := f.Write(m.Body)
if err != nil {
f.logf(lg.FATAL, "[%s/%s] writing message to disk: %s", f.topic, f.opts.Channel, err)
os.Exit(1)
}
_, err = f.Write([]byte("\n"))
if err != nil {
f.logf(lg.FATAL, "[%s/%s] writing newline to disk: %s", f.topic, f.opts.Channel, err)
os.Exit(1)
}
output[pos] = m
pos++
if pos == cap(output) {
sync = true
}
}
if sync || f.consumer.IsStarved() {
if pos > 0 {
f.logf(lg.INFO, "[%s/%s] syncing %d records to disk", f.topic, f.opts.Channel, pos)
err := f.Sync()
if err != nil {
f.logf(lg.FATAL, "[%s/%s] failed syncing messages: %s", f.topic, f.opts.Channel, err)
os.Exit(1)
}
for pos > 0 {
pos--
m := output[pos]
m.Finish()
output[pos] = nil
}
}
sync = false
}
if closeFile {
f.Close()
closeFile = false
}
if exit {
break
}
}
}
func (f *FileLogger) Close() {
if f.out == nil {
return
}
if f.gzipWriter != nil {
err := f.gzipWriter.Close()
if err != nil {
f.logf(lg.FATAL, "[%s/%s] failed to close GZIP writer: %s", f.topic, f.opts.Channel, err)
os.Exit(1)
}
}
err := f.out.Sync()
if err != nil {
f.logf(lg.FATAL, "[%s/%s] failed to fsync output file: %s", f.topic, f.opts.Channel, err)
os.Exit(1)
}
err = f.out.Close()
if err != nil {
f.logf(lg.FATAL, "[%s/%s] failed to close output file: %s", f.topic, f.opts.Channel, err)
os.Exit(1)
}
// Move file from work dir to output dir if necessary, taking care not
// to overwrite existing files
if f.opts.WorkDir != f.opts.OutputDir {
src := f.out.Name()
dst := filepath.Join(f.opts.OutputDir, strings.TrimPrefix(src, f.opts.WorkDir))
// Optimistic rename
f.logf(lg.INFO, "[%s/%s] moving finished file %s to %s", f.topic, f.opts.Channel, src, dst)
err := exclusiveRename(src, dst)
if err == nil {
return
} else if !os.IsExist(err) {
f.logf(lg.FATAL, "[%s/%s] unable to move file from %s to %s: %s", f.topic, f.opts.Channel, src, dst, err)
os.Exit(1)
}
// Optimistic rename failed, so we need to generate a new
// destination file name by bumping the revision number.
_, filenameTmpl := filepath.Split(f.filename)
dstDir, _ := filepath.Split(dst)
dstTmpl := filepath.Join(dstDir, filenameTmpl)
for i := f.rev + 1; ; i++ {
f.logf(lg.WARN, "[%s/%s] destination file already exists: %s", f.topic, f.opts.Channel, dst)
dst := strings.Replace(dstTmpl, "<REV>", fmt.Sprintf("-%06d", i), -1)
err := exclusiveRename(src, dst)
if err != nil {
if os.IsExist(err) {
continue // next rev
}
f.logf(lg.FATAL, "[%s/%s] unable to rename file from %s to %s: %s", f.topic, f.opts.Channel, src, dst, err)
os.Exit(1)
}
f.logf(lg.INFO, "[%s/%s] renamed finished file %s to %s to avoid overwrite", f.topic, f.opts.Channel, src, dst)
break
}
}
f.out = nil
}
func (f *FileLogger) Write(p []byte) (int, error) {
n, err := f.writer.Write(p)
f.filesize += int64(n)
return n, err
}
func (f *FileLogger) Sync() error {
var err error
if f.gzipWriter != nil {
// finish current gzip stream and start a new one (concatenated)
// gzip stream trailer has checksum, and can indicate which messages were ACKed
err = f.gzipWriter.Close()
if err != nil {
return err
}
err = f.out.Sync()
f.gzipWriter, _ = gzip.NewWriterLevel(f.out, f.opts.GZIPLevel)
f.writer = f.gzipWriter
} else {
err = f.out.Sync()
}
return err
}
func (f *FileLogger) currentFilename() string {
t := time.Now()
datetime := strftime(f.opts.DatetimeFormat, t)
return strings.Replace(f.filenameFormat, "<DATETIME>", datetime, -1)
}
func (f *FileLogger) needsRotation() bool {
if f.out == nil {
return true
}
filename := f.currentFilename()
if filename != f.filename {
f.logf(lg.INFO, "[%s/%s] new filename %s, rotating...", f.topic, f.opts.Channel, filename)
return true // rotate by filename
}
if f.opts.RotateInterval > 0 {
if s := time.Since(f.openTime); s > f.opts.RotateInterval {
f.logf(lg.INFO, "[%s/%s] %s since last open, rotating...", f.topic, f.opts.Channel, s)
return true // rotate by interval
}
}
if f.opts.RotateSize > 0 && f.filesize > f.opts.RotateSize {
f.logf(lg.INFO, "[%s/%s] %s currently %d bytes (> %d), rotating...",
f.topic, f.opts.Channel, f.out.Name(), f.filesize, f.opts.RotateSize)
return true // rotate by size
}
return false
}
func (f *FileLogger) updateFile() {
f.Close() // uses current f.filename and f.rev to resolve rename dst conflict
filename := f.currentFilename()
if filename != f.filename {
f.rev = 0 // reset revision to 0 if it is a new filename
} else {
f.rev++
}
f.filename = filename
f.openTime = time.Now()
fullPath := path.Join(f.opts.WorkDir, filename)
err := makeDirFromPath(f.logf, fullPath)
if err != nil {
f.logf(lg.FATAL, "[%s/%s] unable to create dir: %s", f.topic, f.opts.Channel, err)
os.Exit(1)
}
var fi os.FileInfo
for ; ; f.rev++ {
absFilename := strings.Replace(fullPath, "<REV>", fmt.Sprintf("-%06d", f.rev), -1)
// If we're using a working directory for in-progress files,
// proactively check for duplicate file names in the output dir to
// prevent conflicts on rename in the normal case
if f.opts.WorkDir != f.opts.OutputDir {
outputFileName := filepath.Join(f.opts.OutputDir, strings.TrimPrefix(absFilename, f.opts.WorkDir))
err := makeDirFromPath(f.logf, outputFileName)
if err != nil {
f.logf(lg.FATAL, "[%s/%s] unable to create dir: %s", f.topic, f.opts.Channel, err)
os.Exit(1)
}
_, err = os.Stat(outputFileName)
if err == nil {
f.logf(lg.WARN, "[%s/%s] output file already exists: %s", f.topic, f.opts.Channel, outputFileName)
continue // next rev
} else if !os.IsNotExist(err) {
f.logf(lg.FATAL, "[%s/%s] unable to stat output file %s: %s", f.topic, f.opts.Channel, outputFileName, err)
os.Exit(1)
}
}
openFlag := os.O_WRONLY | os.O_CREATE
if f.opts.GZIP || f.opts.RotateInterval > 0 {
openFlag |= os.O_EXCL
} else {
openFlag |= os.O_APPEND
}
f.out, err = os.OpenFile(absFilename, openFlag, 0666)
if err != nil {
if os.IsExist(err) {
f.logf(lg.WARN, "[%s/%s] working file already exists: %s", f.topic, f.opts.Channel, absFilename)
continue // next rev
}
f.logf(lg.FATAL, "[%s/%s] unable to open %s: %s", f.topic, f.opts.Channel, absFilename, err)
os.Exit(1)
}
f.logf(lg.INFO, "[%s/%s] opening %s", f.topic, f.opts.Channel, absFilename)
fi, err = f.out.Stat()
if err != nil {
f.logf(lg.FATAL, "[%s/%s] unable to stat file %s: %s", f.topic, f.opts.Channel, f.out.Name(), err)
}
f.filesize = fi.Size()
if f.opts.RotateSize > 0 && f.filesize > f.opts.RotateSize {
f.logf(lg.INFO, "[%s/%s] %s currently %d bytes (> %d), rotating...",
f.topic, f.opts.Channel, f.out.Name(), f.filesize, f.opts.RotateSize)
continue // next rev
}
break // good file
}
if f.opts.GZIP {
f.gzipWriter, _ = gzip.NewWriterLevel(f.out, f.opts.GZIPLevel)
f.writer = f.gzipWriter
} else {
f.writer = f.out
}
}
func makeDirFromPath(logf lg.AppLogFunc, path string) error {
dir, _ := filepath.Split(path)
if dir != "" {
return os.MkdirAll(dir, 0770)
}
return nil
}
func exclusiveRename(src, dst string) error {
err := os.Link(src, dst)
if err != nil {
return err
}
err = os.Remove(src)
if err != nil {
return err
}
return nil
}
func computeFilenameFormat(opts *Options, topic string) (string, error) {
hostname, err := os.Hostname()
if err != nil {
return "", err
}
shortHostname := strings.Split(hostname, ".")[0]
identifier := shortHostname
if len(opts.HostIdentifier) != 0 {
identifier = strings.Replace(opts.HostIdentifier, "<SHORT_HOST>", shortHostname, -1)
identifier = strings.Replace(identifier, "<HOSTNAME>", hostname, -1)
}
cff := opts.FilenameFormat
if opts.GZIP || opts.RotateSize > 0 || opts.RotateInterval > 0 || opts.WorkDir != opts.OutputDir {
if !strings.Contains(cff, "<REV>") {
return "", errors.New("missing <REV> in --filename-format when gzip, rotation, or work dir enabled")
}
} else {
// remove <REV> as we don't need it
cff = strings.Replace(cff, "<REV>", "", -1)
}
cff = strings.Replace(cff, "<TOPIC>", topic, -1)
cff = strings.Replace(cff, "<HOST>", identifier, -1)
cff = strings.Replace(cff, "<PID>", fmt.Sprintf("%d", os.Getpid()), -1)
if opts.GZIP && !strings.HasSuffix(cff, ".gz") {
cff = cff + ".gz"
}
return cff, nil
}
================================================
FILE: apps/nsq_to_file/nsq_to_file.go
================================================
// This is a client that writes out to a file, and optionally rolls the file
package main
import (
"flag"
"fmt"
"log"
"os"
"os/signal"
"syscall"
"time"
"github.com/mreiferson/go-options"
"github.com/nsqio/go-nsq"
"github.com/nsqio/nsq/internal/app"
"github.com/nsqio/nsq/internal/lg"
"github.com/nsqio/nsq/internal/version"
)
func flagSet() *flag.FlagSet {
fs := flag.NewFlagSet("nsqd", flag.ExitOnError)
fs.Bool("version", false, "print version string")
fs.String("log-level", "info", "set log verbosity: debug, info, warn, error, or fatal")
fs.String("log-prefix", "[nsq_to_file] ", "log message prefix")
fs.String("channel", "nsq_to_file", "nsq channel")
fs.Int("max-in-flight", 200, "max number of messages to allow in flight")
fs.String("output-dir", "/tmp", "directory to write output files to")
fs.String("work-dir", "", "directory for in-progress files before moving to output-dir")
fs.String("datetime-format", "%Y-%m-%d_%H", "strftime compatible format for <DATETIME> in filename format")
fs.String("filename-format", "<TOPIC>.<HOST><REV>.<DATETIME>.log", "output filename format (<TOPIC>, <HOST>, <PID>, <DATETIME>, <REV> are replaced. <REV> is increased when file already exists)")
fs.String("host-identifier", "", "value to output in log filename in place of hostname. <SHORT_HOST> and <HOSTNAME> are valid replacement tokens")
fs.Int("gzip-level", 6, "gzip compression level (1-9, 1=BestSpeed, 9=BestCompression)")
fs.Bool("gzip", false, "gzip output files.")
fs.Bool("skip-empty-files", false, "skip writing empty files")
fs.Duration("topic-refresh", time.Minute, "how frequently the topic list should be refreshed")
fs.String("topic-pattern", "", "only log topics matching the following pattern")
fs.Int64("rotate-size", 0, "rotate the file when it grows bigger than `rotate-size` bytes")
fs.Duration("rotate-interval", 0, "rotate the file every duration")
fs.Duration("sync-interval", 30*time.Second, "sync file to disk every duration")
fs.Duration("http-client-connect-timeout", 2*time.Second, "timeout for HTTP connect")
fs.Duration("http-client-request-timeout", 5*time.Second, "timeout for HTTP request")
nsqdTCPAddrs := app.StringArray{}
lookupdHTTPAddrs := app.StringArray{}
topics := app.StringArray{}
consumerOpts := app.StringArray{}
fs.Var(&nsqdTCPAddrs, "nsqd-tcp-address", "nsqd TCP address (may be given multiple times)")
fs.Var(&lookupdHTTPAddrs, "lookupd-http-address", "lookupd HTTP address (may be given multiple times)")
fs.Var(&topics, "topic", "nsq topic (may be given multiple times)")
fs.Var(&consumerOpts, "consumer-opt", "option to passthrough to nsq.Consumer (may be given multiple times, http://godoc.org/github.com/nsqio/go-nsq#Config)")
return fs
}
func main() {
fs := flagSet()
fs.Parse(os.Args[1:])
if args := fs.Args(); len(args) > 0 {
log.Fatalf("unknown arguments: %s", args)
}
opts := NewOptions()
options.Resolve(opts, fs, nil)
logger := log.New(os.Stderr, opts.LogPrefix, log.Ldate|log.Ltime|log.Lmicroseconds)
logLevel, err := lg.ParseLogLevel(opts.LogLevel)
if err != nil {
log.Fatal("--log-level is invalid")
}
logf := func(lvl lg.LogLevel, f string, args ...interface{}) {
lg.Logf(logger, logLevel, lvl, f, args...)
}
if fs.Lookup("version").Value.(flag.Getter).Get().(bool) {
fmt.Printf("nsq_to_file v%s\n", version.Binary)
return
}
if opts.Channel == "" {
log.Fatal("--channel is required")
}
if opts.HTTPClientConnectTimeout <= 0 {
log.Fatal("--http-client-connect-timeout should be positive")
}
if opts.HTTPClientRequestTimeout <= 0 {
log.Fatal("--http-client-request-timeout should be positive")
}
if len(opts.NSQDTCPAddrs) == 0 && len(opts.NSQLookupdHTTPAddrs) == 0 {
log.Fatal("--nsqd-tcp-address or --lookupd-http-address required.")
}
if len(opts.NSQDTCPAddrs) != 0 && len(opts.NSQLookupdHTTPAddrs) != 0 {
log.Fatal("use --nsqd-tcp-address or --lookupd-http-address not both")
}
if opts.GZIPLevel < 1 || opts.GZIPLevel > 9 {
log.Fatalf("invalid --gzip-level value (%d), should be 1-9", opts.GZIPLevel)
}
if len(opts.Topics) == 0 && len(opts.TopicPattern) == 0 {
log.Fatal("--topic or --topic-pattern required")
}
if len(opts.Topics) == 0 && len(opts.NSQLookupdHTTPAddrs) == 0 {
log.Fatal("--lookupd-http-address must be specified when no --topic specified")
}
if opts.WorkDir == "" {
opts.WorkDir = opts.OutputDir
}
cfg := nsq.NewConfig()
cfgFlag := nsq.ConfigFlag{cfg}
for _, opt := range opts.ConsumerOpts {
cfgFlag.Set(opt)
}
cfg.UserAgent = fmt.Sprintf("nsq_to_file/%s go-nsq/%s", version.Binary, nsq.VERSION)
cfg.MaxInFlight = opts.MaxInFlight
hupChan := make(chan os.Signal, 1)
termChan := make(chan os.Signal, 1)
signal.Notify(hupChan, syscall.SIGHUP)
signal.Notify(termChan, syscall.SIGINT, syscall.SIGTERM)
discoverer := newTopicDiscoverer(logf, opts, cfg, hupChan, termChan)
discoverer.run()
}
================================================
FILE: apps/nsq_to_file/options.go
================================================
package main
import "time"
type Options struct {
Topics []string `flag:"topic"`
TopicPattern string `flag:"topic-pattern"`
TopicRefreshInterval time.Duration `flag:"topic-refresh"`
Channel string `flag:"channel"`
NSQDTCPAddrs []string `flag:"nsqd-tcp-address"`
NSQLookupdHTTPAddrs []string `flag:"lookupd-http-address"`
ConsumerOpts []string `flag:"consumer-opt"`
MaxInFlight int `flag:"max-in-flight"`
HTTPClientConnectTimeout time.Duration `flag:"http-client-connect-timeout"`
HTTPClientRequestTimeout time.Duration `flag:"http-client-request-timeout"`
LogPrefix string `flag:"log-prefix"`
LogLevel string `flag:"log-level"`
OutputDir string `flag:"output-dir"`
WorkDir string `flag:"work-dir"`
DatetimeFormat string `flag:"datetime-format"`
FilenameFormat string `flag:"filename-format"`
HostIdentifier string `flag:"host-identifier"`
GZIPLevel int `flag:"gzip-level"`
GZIP bool `flag:"gzip"`
SkipEmptyFiles bool `flag:"skip-empty-files"`
RotateSize int64 `flag:"rotate-size"`
RotateInterval time.Duration `flag:"rotate-interval"`
SyncInterval time.Duration `flag:"sync-interval"`
}
func NewOptions() *Options {
return &Options{
LogPrefix: "[nsq_to_file] ",
LogLevel: "info",
Channel: "nsq_to_file",
MaxInFlight: 200,
OutputDir: "/tmp",
DatetimeFormat: "%Y-%m-%d_%H",
FilenameFormat: "<TOPIC>.<HOST><REV>.<DATETIME>.log",
GZIPLevel: 6,
TopicRefreshInterval: time.Minute,
SyncInterval: 30 * time.Second,
HTTPClientConnectTimeout: 2 * time.Second,
HTTPClientRequestTimeout: 5 * time.Second,
}
}
================================================
FILE: apps/nsq_to_file/strftime.go
================================================
// COPIED FROM https://github.com/jehiah/go-strftime
package main
import (
"time"
)
// taken from time/format.go
var conversion = map[string]string{
/*stdLongMonth */ "B": "January",
/*stdMonth */ "b": "Jan",
// stdNumMonth */ "m": "1",
/*stdZeroMonth */ "m": "01",
/*stdLongWeekDay */ "A": "Monday",
/*stdWeekDay */ "a": "Mon",
// stdDay */ "d": "2",
// stdUnderDay */ "d": "_2",
/*stdZeroDay */ "d": "02",
/*stdHour */ "H": "15",
// stdHour12 */ "I": "3",
/*stdZeroHour12 */ "I": "03",
// stdMinute */ "M": "4",
/*stdZeroMinute */ "M": "04",
// stdSecond */ "S": "5",
/*stdZeroSecond */ "S": "05",
/*stdLongYear */ "Y": "2006",
/*stdYear */ "y": "06",
/*stdPM */ "p": "PM",
// stdpm */ "p": "pm",
/*stdTZ */ "Z": "MST",
// stdISO8601TZ */ "z": "Z0700", // prints Z for UTC
// stdISO8601ColonTZ */ "z": "Z07:00", // prints Z for UTC
/*stdNumTZ */ "z": "-0700", // always numeric
// stdNumShortTZ */ "b": "-07", // always numeric
// stdNumColonTZ */ "b": "-07:00", // always numeric
"%": "%",
}
// This is an alternative to time.Format because no one knows
// what date 040305 is supposed to create when used as a 'layout' string
// this takes standard strftime format options. For a complete list
// of format options see http://strftime.org/
func strftime(format string, t time.Time) string {
layout := ""
length := len(format)
for i := 0; i < length; i++ {
if format[i] == '%' && i <= length-2 {
if layoutCmd, ok := conversion[format[i+1:i+2]]; ok {
layout = layout + layoutCmd
i++
continue
}
}
layout = layout + format[i:i+1]
}
return t.Format(layout)
}
================================================
FILE: apps/nsq_to_file/topic_discoverer.go
================================================
package main
import (
"os"
"regexp"
"sync"
"time"
"github.com/nsqio/go-nsq"
"github.com/nsqio/nsq/internal/clusterinfo"
"github.com/nsqio/nsq/internal/http_api"
"github.com/nsqio/nsq/internal/lg"
)
type TopicDiscoverer struct {
logf lg.AppLogFunc
opts *Options
ci *clusterinfo.ClusterInfo
topics map[string]*FileLogger
hupChan chan os.Signal
termChan chan os.Signal
wg sync.WaitGroup
cfg *nsq.Config
}
func newTopicDiscoverer(logf lg.AppLogFunc, opts *Options, cfg *nsq.Config, hupChan chan os.Signal, termChan chan os.Signal) *TopicDiscoverer {
client := http_api.NewClient(nil, opts.HTTPClientConnectTimeout, opts.HTTPClientRequestTimeout)
return &TopicDiscoverer{
logf: logf,
opts: opts,
ci: clusterinfo.New(nil, client),
topics: make(map[string]*FileLogger),
hupChan: hupChan,
termChan: termChan,
cfg: cfg,
}
}
func (t *TopicDiscoverer) updateTopics(topics []string) {
for _, topic := range topics {
if _, ok := t.topics[topic]; ok {
continue
}
if !t.isTopicAllowed(topic) {
t.logf(lg.WARN, "skipping topic %s (doesn't match pattern %s)", topic, t.opts.TopicPattern)
continue
}
fl, err := NewFileLogger(t.logf, t.opts, topic, t.cfg)
if err != nil {
t.logf(lg.ERROR, "couldn't create logger for new topic %s: %s", topic, err)
continue
}
t.topics[topic] = fl
t.wg.Add(1)
go func(fl *FileLogger) {
fl.router()
t.wg.Done()
}(fl)
}
}
func (t *TopicDiscoverer) run() {
var ticker <-chan time.Time
if len(t.opts.Topics) == 0 {
ticker = time.Tick(t.opts.TopicRefreshInterval)
}
t.updateTopics(t.opts.Topics)
forloop:
for {
select {
case <-ticker:
newTopics, err := t.ci.GetLookupdTopics(t.opts.NSQLookupdHTTPAddrs)
if err != nil {
t.logf(lg.ERROR, "could not retrieve topic list: %s", err)
continue
}
t.updateTopics(newTopics)
case <-t.termChan:
for _, fl := range t.topics {
close(fl.termChan)
}
break forloop
case <-t.hupChan:
for _, fl := range t.topics {
fl.hupChan <- true
}
}
}
t.wg.Wait()
}
func (t *TopicDiscoverer) isTopicAllowed(topic string) bool {
if t.opts.TopicPattern == "" {
return true
}
match, err := regexp.MatchString(t.opts.TopicPattern, topic)
if err != nil {
return false
}
return match
}
================================================
FILE: apps/nsq_to_http/http.go
================================================
package main
import (
"bytes"
"fmt"
"net/http"
"github.com/nsqio/nsq/internal/version"
)
var httpclient *http.Client
var userAgent string
func init() {
userAgent = fmt.Sprintf("nsq_to_http v%s", version.Binary)
}
func HTTPGet(endpoint string) (*http.Response, error) {
req, err := http.NewRequest("GET", endpoint, nil)
if err != nil {
return nil, err
}
req.Header.Set("User-Agent", userAgent)
for key, val := range validCustomHeaders {
req.Header.Set(key, val)
}
return httpclient.Do(req)
}
func HTTPPost(endpoint string, body *bytes.Buffer) (*http.Response, error) {
req, err := http.NewRequest("POST", endpoint, body)
if err != nil {
return nil, err
}
req.Header.Set("User-Agent", userAgent)
req.Header.Set("Content-Type", *contentType)
for key, val := range validCustomHeaders {
req.Header.Set(key, val)
}
return httpclient.Do(req)
}
================================================
FILE: apps/nsq_to_http/nsq_to_http.go
================================================
// This is an NSQ client that reads the specified topic/channel
// and performs HTTP requests (GET/POST) to the specified endpoints
package main
import (
"bytes"
"flag"
"fmt"
"io"
"log"
"math/rand"
"net/http"
"net/url"
"os"
"os/signal"
"strings"
"sync/atomic"
"syscall"
"time"
"github.com/bitly/go-hostpool"
"github.com/bitly/timer_metrics"
"github.com/nsqio/go-nsq"
"github.com/nsqio/nsq/internal/app"
"github.com/nsqio/nsq/internal/http_api"
"github.com/nsqio/nsq/internal/version"
)
const (
ModeAll = iota
ModeRoundRobin
ModeHostPool
)
var (
showVersion = flag.Bool("version", false, "print version string")
topic = flag.String("topic", "", "nsq topic")
channel = flag.String("channel", "nsq_to_http", "nsq channel")
maxInFlight = flag.Int("max-in-flight", 200, "max number of messages to allow in flight")
numPublishers = flag.Int("n", 100, "number of concurrent publishers")
mode = flag.String("mode", "hostpool", "the upstream request mode options: round-robin, hostpool (default), epsilon-greedy")
sample = flag.Float64("sample", 1.0, "% of messages to publish (float b/w 0 -> 1)")
httpConnectTimeout = flag.Duration("http-client-connect-timeout", 2*time.Second, "timeout for HTTP connect")
httpRequestTimeout = flag.Duration("http-client-request-timeout", 20*time.Second, "timeout for HTTP request")
statusEvery = flag.Int("status-every", 250, "the # of requests between logging status (per handler), 0 disables")
contentType = flag.String("content-type", "application/octet-stream", "the Content-Type used for POST requests")
getAddrs = app.StringArray{}
postAddrs = app.StringArray{}
customHeaders = app.StringArray{}
nsqdTCPAddrs = app.StringArray{}
lookupdHTTPAddrs = app.StringArray{}
validCustomHeaders map[string]string
)
func init() {
flag.Var(&postAddrs, "post", "HTTP address to make a POST request to. data will be in the body (may be given multiple times)")
flag.Var(&customHeaders, "header", "Custom header for HTTP requests (may be given multiple times)")
flag.Var(&getAddrs, "get", "HTTP address to make a GET request to. '%s' will be printf replaced with data (may be given multiple times)")
flag.Var(&nsqdTCPAddrs, "nsqd-tcp-address", "nsqd TCP address (may be given multiple times)")
flag.Var(&lookupdHTTPAddrs, "lookupd-http-address", "lookupd HTTP address (may be given multiple times)")
}
type Publisher interface {
Publish(string, []byte) error
}
type PublishHandler struct {
// 64bit atomic vars need to be first for proper alignment on 32bit platforms
counter uint64
Publisher
addresses app.StringArray
mode int
hostPool hostpool.HostPool
perAddressStatus map[string]*timer_metrics.TimerMetrics
timermetrics *timer_metrics.TimerMetrics
}
func (ph *PublishHandler) HandleMessage(m *nsq.Message) error {
if *sample < 1.0 && rand.Float64() > *sample {
return nil
}
startTime := time.Now()
switch ph.mode {
case ModeAll:
for _, addr := range ph.addresses {
st := time.Now()
err := ph.Publish(addr, m.Body)
if err != nil {
return err
}
ph.perAddressStatus[addr].Status(st)
}
case ModeRoundRobin:
counter := atomic.AddUint64(&ph.counter, 1)
idx := counter % uint64(len(ph.addresses))
addr := ph.addresses[idx]
err := ph.Publish(addr, m.Body)
if err != nil {
return err
}
ph.perAddressStatus[addr].Status(startTime)
case ModeHostPool:
hostPoolResponse := ph.hostPool.Get()
addr := hostPoolResponse.Host()
err := ph.Publish(addr, m.Body)
hostPoolResponse.Mark(err)
if err != nil {
return err
}
ph.perAddressStatus[addr].Status(startTime)
}
ph.timermetrics.Status(startTime)
return nil
}
type PostPublisher struct{}
func (p *PostPublisher) Publish(addr string, msg []byte) error {
buf := bytes.NewBuffer(msg)
resp, err := HTTPPost(addr, buf)
if err != nil {
return err
}
io.Copy(io.Discard, resp.Body)
resp.Body.Close()
if resp.StatusCode < 200 || resp.StatusCode >= 300 {
return fmt.Errorf("got status code %d", resp.StatusCode)
}
return nil
}
type GetPublisher struct{}
func (p *GetPublisher) Publish(addr string, msg []byte) error {
endpoint := fmt.Sprintf(addr, url.QueryEscape(string(msg)))
resp, err := HTTPGet(endpoint)
if err != nil {
return err
}
io.Copy(io.Discard, resp.Body)
resp.Body.Close()
if resp.StatusCode != 200 {
return fmt.Errorf("got status code %d", resp.StatusCode)
}
return nil
}
func main() {
var publisher Publisher
var addresses app.StringArray
var selectedMode int
cfg := nsq.NewConfig()
flag.Var(&nsq.ConfigFlag{cfg}, "consumer-opt", "option to passthrough to nsq.Consumer (may be given multiple times, http://godoc.org/github.com/nsqio/go-nsq#Config)")
flag.Parse()
httpclient = &http.Client{Transport: http_api.NewDeadlineTransport(*httpConnectTimeout, *httpRequestTimeout), Timeout: *httpRequestTimeout}
if *showVersion {
fmt.Printf("nsq_to_http v%s\n", version.Binary)
return
}
if len(customHeaders) > 0 {
var err error
validCustomHeaders, err = parseCustomHeaders(customHeaders)
if err != nil {
log.Fatal("--header value format should be 'key=value'")
}
}
if *topic == "" || *channel == "" {
log.Fatal("--topic and --channel are required")
}
if *contentType != flag.Lookup("content-type").DefValue {
if len(postAddrs) == 0 {
log.Fatal("--content-type only used with --post")
}
if len(*contentType) == 0 {
log.Fatal("--content-type requires a value when used")
}
}
if len(nsqdTCPAddrs) == 0 && len(lookupdHTTPAddrs) == 0 {
log.Fatal("--nsqd-tcp-address or --lookupd-http-address required")
}
if len(nsqdTCPAddrs) > 0 && len(lookupdHTTPAddrs) > 0 {
log.Fatal("use --nsqd-tcp-address or --lookupd-http-address not both")
}
if len(getAddrs) == 0 && len(postAddrs) == 0 {
log.Fatal("--get or --post required")
}
if len(getAddrs) > 0 && len(postAddrs) > 0 {
log.Fatal("use --get or --post not both")
}
if len(getAddrs) > 0 {
for _, get := range getAddrs {
if strings.Count(get, "%s") != 1 {
log.Fatal("invalid GET address - must be a printf string")
}
}
}
switch *mode {
case "round-robin":
selectedMode = ModeRoundRobin
case "hostpool", "epsilon-greedy":
selectedMode = ModeHostPool
}
if *sample > 1.0 || *sample < 0.0 {
log.Fatal("ERROR: --sample must be between 0.0 and 1.0")
}
termChan := make(chan os.Signal, 1)
signal.Notify(termChan, syscall.SIGINT, syscall.SIGTERM)
if len(postAddrs) > 0 {
publisher = &PostPublisher{}
addresses = postAddrs
} else {
publisher = &GetPublisher{}
addresses = getAddrs
}
cfg.UserAgent = fmt.Sprintf("nsq_to_http/%s go-nsq/%s", version.Binary, nsq.VERSION)
cfg.MaxInFlight = *maxInFlight
consumer, err := nsq.NewConsumer(*topic, *channel, cfg)
if err != nil {
log.Fatal(err)
}
perAddressStatus := make(map[string]*timer_metrics.TimerMetrics)
if len(addresses) == 1 {
// disable since there is only one address
perAddressStatus[addresses[0]] = timer_metrics.NewTimerMetrics(0, "")
} else {
for _, a := range addresses {
perAddressStatus[a] = timer_metrics.NewTimerMetrics(*statusEvery,
fmt.Sprintf("[%s]:", a))
}
}
hostPool := hostpool.New(addresses)
if *mode == "epsilon-greedy" {
hostPool = hostpool.NewEpsilonGreedy(addresses, 0, &hostpool.LinearEpsilonValueCalculator{})
}
handler := &PublishHandler{
Publisher: publisher,
addresses: addresses,
mode: selectedMode,
hostPool: hostPool,
perAddressStatus: perAddressStatus,
timermetrics: timer_metrics.NewTimerMetrics(*statusEvery, "[aggregate]:"),
}
consumer.AddConcurrentHandlers(handler, *numPublishers)
err = consumer.ConnectToNSQDs(nsqdTCPAddrs)
if err != nil {
log.Fatal(err)
}
err = consumer.ConnectToNSQLookupds(lookupdHTTPAddrs)
if err != nil {
log.Fatal(err)
}
for {
select {
case <-consumer.StopChan:
return
case <-termChan:
consumer.Stop()
}
}
}
func parseCustomHeaders(strs []string) (map[string]string, error) {
parsedHeaders := make(map[string]string)
for _, s := range strs {
sp := strings.SplitN(s, ":", 2)
if len(sp) != 2 {
return nil, fmt.Errorf("invalid header: %q", s)
}
key := strings.TrimSpace(sp[0])
val := strings.TrimSpace(sp[1])
if key == "" || val == "" {
return nil, fmt.Errorf("invalid header: %q", s)
}
parsedHeaders[key] = val
}
return parsedHeaders, nil
}
================================================
FILE: apps/nsq_to_http/nsq_to_http_test.go
================================================
// This is an NSQ client that reads the specified topic/channel
// and performs HTTP requests (GET/POST) to the specified endpoints
package main
import (
"reflect"
"testing"
)
func TestParseCustomHeaders(t *testing.T) {
type args struct {
strs []string
}
tests := []struct {
name string
args args
want map[string]string
wantErr bool
}{
{
"Valid Custom Headers",
args{[]string{"header1: value1", "header2:value2", "header3:value3", "header4:value4"}},
map[string]string{"header1": "value1", "header2": "value2", "header3": "value3", "header4": "value4"},
false,
},
{
"Invalid Custom Headers where key is present but no value",
args{[]string{"header1:", "header2:value2", "header3: value3", "header4:value4"}},
nil,
true,
},
{
"Invalid Custom Headers where key is not present but value is present",
args{[]string{"header1: value1", ": value2", "header3:value3", "header4:value4"}},
nil,
true,
},
{
"Invalid Custom Headers where key and value are not present but ':' is specified",
args{[]string{"header1:value1", "header2:value2", ":", "header4:value4"}},
nil,
true,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
got, err := parseCustomHeaders(tt.args.strs)
if (err != nil) != tt.wantErr {
t.Errorf("parseCustomHeaders() error = %v, wantErr %v", err, tt.wantErr)
return
}
if !reflect.DeepEqual(got, tt.want) {
t.Errorf("parseCustomHeaders() = %v, want %v", got, tt.want)
}
})
}
}
================================================
FILE: apps/nsq_to_nsq/nsq_to_nsq.go
================================================
// This is an NSQ client that reads the specified topic/channel
// and re-publishes the messages to destination nsqd via TCP
package main
import (
"encoding/json"
"errors"
"flag"
"fmt"
"log"
"os"
"os/signal"
"strconv"
"sync/atomic"
"syscall"
"time"
"github.com/bitly/go-hostpool"
"github.com/bitly/timer_metrics"
"github.com/nsqio/go-nsq"
"github.com/nsqio/nsq/internal/app"
"github.com/nsqio/nsq/internal/protocol"
"github.com/nsqio/nsq/internal/version"
)
const (
ModeRoundRobin = iota
ModeHostPool
)
var (
showVersion = flag.Bool("version", false, "print version string")
channel = flag.String("channel", "nsq_to_nsq", "nsq channel")
destTopic = flag.String("destination-topic", "", "use this destination topic for all consumed topics (default is consumed topic name)")
maxInFlight = flag.Int("max-in-flight", 200, "max number of messages to allow in flight")
statusEvery = flag.Int("status-every", 250, "the # of requests between logging status (per destination), 0 disables")
mode = flag.String("mode", "hostpool", "the upstream request mode options: round-robin, hostpool (default), epsilon-greedy")
nsqdTCPAddrs = app.StringArray{}
lookupdHTTPAddrs = app.StringArray{}
destNsqdTCPAddrs = app.StringArray{}
whitelistJSONFields = app.StringArray{}
topics = app.StringArray{}
requireJSONField = flag.String("require-json-field", "", "for JSON messages: only pass messages that contain this field")
requireJSONValue = flag.String("require-json-value", "", "for JSON messages: only pass messages in which the required field has this value")
)
func init() {
flag.Var(&nsqdTCPAddrs, "nsqd-tcp-address", "nsqd TCP address (may be given multiple times)")
flag.Var(&destNsqdTCPAddrs, "destination-nsqd-tcp-address", "destination nsqd TCP address (may be given multiple times)")
flag.Var(&lookupdHTTPAddrs, "lookupd-http-address", "lookupd HTTP address (may be given multiple times)")
flag.Var(&topics, "topic", "nsq topic (may be given multiple times)")
flag.Var(&whitelistJSONFields, "whitelist-json-field", "for JSON messages: pass this field (may be given multiple times)")
}
type PublishHandler struct {
// 64bit atomic vars need to be first for proper alignment on 32bit platforms
counter uint64
addresses app.StringArray
producers map[string]*nsq.Producer
mode int
hostPool hostpool.HostPool
respChan chan *nsq.ProducerTransaction
requireJSONValueParsed bool
requireJSONValueIsNumber bool
requireJSONNumber float64
perAddressStatus map[string]*timer_metrics.TimerMetrics
timermetrics *timer_metrics.TimerMetrics
}
type TopicHandler struct {
publishHandler *PublishHandler
destinationTopic string
}
func (ph *PublishHandler) responder() {
var msg *nsq.Message
var startTime time.Time
var address string
var hostPoolResponse hostpool.HostPoolResponse
for t := range ph.respChan {
switch ph.mode {
case ModeRoundRobin:
msg = t.Args[0].(*nsq.Message)
startTime = t.Args[1].(time.Time)
hostPoolResponse = nil
address = t.Args[2].(string)
case ModeHostPool:
msg = t.Args[0].(*nsq.Message)
startTime = t.Args[1].(time.Time)
hostPoolResponse = t.Args[2].(hostpool.HostPoolResponse)
address = hostPoolResponse.Host()
}
success := t.Error == nil
if hostPoolResponse != nil {
if !success {
hostPoolResponse.Mark(errors.New("failed"))
} else {
hostPoolResponse.Mark(nil)
}
}
if success {
msg.Finish()
} else {
msg.Requeue(-1)
}
ph.perAddressStatus[address].Status(startTime)
ph.timermetrics.Status(startTime)
}
}
func (ph *PublishHandler) shouldPassMessage(js map[string]interface{}) (bool, bool) {
pass := true
backoff := false
if *requireJSONField == "" {
return pass, backoff
}
if *requireJSONValue != "" && !ph.requireJSONValueParsed {
// cache conversion in case needed while filtering json
var err error
ph.requireJSONNumber, err = strconv.ParseFloat(*requireJSONValue, 64)
ph.requireJSONValueIsNumber = (err == nil)
ph.requireJSONValueParsed = true
}
v, ok := js[*requireJSONField]
if !ok {
pass = false
if *requireJSONValue != "" {
log.Printf("ERROR: missing field to check required value")
backoff = true
}
} else if *requireJSONValue != "" {
// if command-line argument can't convert to float, then it can't match a number
// if it can, also integers (up to 2^53 or so) can be compared as float64
if s, ok := v.(string); ok {
if s != *requireJSONValue {
pass = false
}
} else if ph.requireJSONValueIsNumber {
f, ok := v.(float64)
if !ok || f != ph.requireJSONNumber {
pass = false
}
} else {
// json value wasn't a plain string, and argument wasn't a number
// give up on comparisons of other types
pass = false
}
}
return pass, backoff
}
func filterMessage(js map[string]interface{}, rawMsg []byte) ([]byte, error) {
if len(whitelistJSONFields) == 0 {
// no change
return rawMsg, nil
}
newMsg := make(map[string]interface{}, len(whitelistJSONFields))
for _, key := range whitelistJSONFields {
value, ok := js[key]
if ok {
// avoid printing int as float (go 1.0)
switch tvalue := value.(type) {
case float64:
ivalue := int64(tvalue)
if float64(ivalue) == tvalue {
newMsg[key] = ivalue
} else {
newMsg[key] = tvalue
}
default:
newMsg[key] = value
}
}
}
newRawMsg, err := json.Marshal(newMsg)
if err != nil {
return nil, fmt.Errorf("unable to marshal filtered message %v", newMsg)
}
return newRawMsg, nil
}
func (t *TopicHandler) HandleMessage(m *nsq.Message) error {
return t.publishHandler.HandleMessage(m, t.destinationTopic)
}
func (ph *PublishHandler) HandleMessage(m *nsq.Message, destinationTopic string) error {
var err error
msgBody := m.Body
if *requireJSONField != "" || len(whitelistJSONFields) > 0 {
var js map[string]interface{}
err = json.Unmarshal(msgBody, &js)
if err != nil {
log.Printf("ERROR: Unable to decode json: %s", msgBody)
return nil
}
if pass, backoff := ph.shouldPassMessage(js); !pass {
if backoff {
return errors.New("backoff")
}
return nil
}
msgBody, err = filterMessage(js, msgBody)
if err != nil {
log.Printf("ERROR: filterMessage() failed: %s", err)
return err
}
}
startTime := time.Now()
switch ph.mode {
case ModeRoundRobin:
counter := atomic.AddUint64(&ph.counter, 1)
idx := counter % uint64(len(ph.addresses))
addr := ph.addresses[idx]
p := ph.producers[addr]
err = p.PublishAsync(destinationTopic, msgBody, ph.respChan, m, startTime, addr)
case ModeHostPool:
hostPoolResponse := ph.hostPool.Get()
p := ph.producers[hostPoolResponse.Host()]
err = p.PublishAsync(destinationTopic, msgBody, ph.respChan, m, startTime, hostPoolResponse)
if err != nil {
hostPoolResponse.Mark(err)
}
}
if err != nil {
return err
}
m.DisableAutoResponse()
return nil
}
func main() {
var selectedMode int
cCfg := nsq.NewConfig()
pCfg := nsq.NewConfig()
flag.Var(&nsq.ConfigFlag{cCfg}, "consumer-opt", "option to passthrough to nsq.Consumer (may be given multiple times, see http://godoc.org/github.com/nsqio/go-nsq#Config)")
flag.Var(&nsq.ConfigFlag{pCfg}, "producer-opt", "option to passthrough to nsq.Producer (may be given multiple times, see http://godoc.org/github.com/nsqio/go-nsq#Config)")
flag.Parse()
if *showVersion {
fmt.Printf("nsq_to_nsq v%s\n", version.Binary)
return
}
if len(topics) == 0 || *channel == "" {
log.Fatal("--topic and --channel are required")
}
for _, topic := range topics {
if !protocol.IsValidTopicName(topic) {
log.Fatal("--topic is invalid")
}
}
if *destTopic != "" && !protocol.IsValidTopicName(*destTopic) {
log.Fatal("--destination-topic is invalid")
}
if !protocol.IsValidChannelName(*channel) {
log.Fatal("--channel is invalid")
}
if len(nsqdTCPAddrs) == 0 && len(lookupdHTTPAddrs) == 0 {
log.Fatal("--nsqd-tcp-address or --lookupd-http-address required")
}
if len(nsqdTCPAddrs) > 0 && len(lookupdHTTPAddrs) > 0 {
log.Fatal("use --nsqd-tcp-address or --lookupd-http-address not both")
}
if len(destNsqdTCPAddrs) == 0 {
log.Fatal("--destination-nsqd-tcp-address required")
}
switch *mode {
case "round-robin":
selectedMode = ModeRoundRobin
case "hostpool", "epsilon-greedy":
selectedMode = ModeHostPool
}
termChan := make(chan os.Signal, 1)
signal.Notify(termChan, syscall.SIGINT, syscall.SIGTERM)
defaultUA := fmt.Sprintf("nsq_to_nsq/%s go-nsq/%s", version.Binary, nsq.VERSION)
cCfg.UserAgent = defaultUA
cCfg.MaxInFlight = *maxInFlight
pCfg.UserAgent = defaultUA
producers := make(map[string]*nsq.Producer)
for _, addr := range destNsqdTCPAddrs {
producer, err := nsq.NewProducer(addr, pCfg)
if err != nil {
log.Fatalf("failed creating producer %s", err)
}
producers[addr] = producer
}
perAddressStatus := make(map[string]*timer_metrics.TimerMetrics)
if len(destNsqdTCPAddrs) == 1 {
// disable since there is only one address
perAddressStatus[destNsqdTCPAddrs[0]] = timer_metrics.NewTimerMetrics(0, "")
} else {
for _, a := range destNsqdTCPAddrs {
perAddressStatus[a] = timer_metrics.NewTimerMetrics(*statusEvery,
fmt.Sprintf("[%s]:", a))
}
}
hostPool := hostpool.New(destNsqdTCPAddrs)
if *mode == "epsilon-greedy" {
hostPool = hostpool.NewEpsilonGreedy(destNsqdTCPAddrs, 0, &hostpool.LinearEpsilonValueCalculator{})
}
var consumerList []*nsq.Consumer
publisher := &PublishHandler{
addresses: destNsqdTCPAddrs,
producers: producers,
mode: selectedMode,
hostPool: hostPool,
respChan: make(chan *nsq.ProducerTransaction, len(destNsqdTCPAddrs)),
perAddressStatus: perAddressStatus,
timermetrics: timer_metrics.NewTimerMetrics(*statusEvery, "[aggregate]:"),
}
for _, topic := range topics {
consumer, err := nsq.NewConsumer(topic, *channel, cCfg)
consumerList = append(consumerList, consumer)
if err != nil {
log.Fatal(err)
}
publishTopic := topic
if *destTopic != "" {
publishTopic = *destTopic
}
topicHandler := &TopicHandler{
publishHandler: publisher,
destinationTopic: publishTopic,
}
consumer.AddConcurrentHandlers(topicHandler, len(destNsqdTCPAddrs))
}
for i := 0; i < len(destNsqdTCPAddrs); i++ {
go publisher.responder()
}
for _, consumer := range consumerList {
err := consumer.ConnectToNSQDs(nsqdTCPAddrs)
if err != nil {
log.Fatal(err)
}
}
for _, consumer := range consumerList {
err := consumer.ConnectToNSQLookupds(lookupdHTTPAddrs)
if err != nil {
log.Fatal(err)
}
}
<-termChan // wait for signal
for _, consumer := range consumerList {
consumer.Stop()
}
for _, consumer := range consumerList {
<-consumer.StopChan
}
}
================================================
FILE: apps/nsqadmin/main.go
================================================
package main
import (
"flag"
"fmt"
"os"
"path/filepath"
"sync"
"syscall"
"github.com/BurntSushi/toml"
"github.com/judwhite/go-svc"
"github.com/mreiferson/go-options"
"github.com/nsqio/nsq/internal/app"
"github.com/nsqio/nsq/internal/lg"
"github.com/nsqio/nsq/internal/version"
"github.com/nsqio/nsq/nsqadmin"
)
func nsqadminFlagSet(opts *nsqadmin.Options) *flag.FlagSet {
flagSet := flag.NewFlagSet("nsqadmin", flag.ExitOnError)
flagSet.String("config", "", "path to config file")
flagSet.Bool("version", false, "print version string")
logLevel := opts.LogLevel
flagSet.Var(&logLevel, "log-level", "set log verbosity: debug, info, warn, error, or fatal")
flagSet.String("log-prefix", "[nsqadmin] ", "log message prefix")
flagSet.Bool("verbose", false, "[deprecated] has no effect, use --log-level")
flagSet.String("http-address", opts.HTTPAddress, "<addr>:<port> to listen on for HTTP clients")
flagSet.String("base-path", opts.BasePath, "URL base path")
flagSet.String("dev-static-dir", opts.DevStaticDir, "(development use only)")
flagSet.String("graphite-url", opts.GraphiteURL, "graphite HTTP address")
flagSet.Bool("proxy-graphite", false, "proxy HTTP requests to graphite")
flagSet.String("statsd-counter-format", opts.StatsdCounterFormat, "The counter stats key formatting applied by the implementation of statsd. If no formatting is desired, set this to an empty string.")
flagSet.String("statsd-gauge-format", opts.StatsdGaugeFormat, "The gauge stats key formatting applied by the implementation of statsd. If no formatting is desired, set this to an empty string.")
flagSet.String("statsd-prefix", opts.StatsdPrefix, "prefix used for keys sent to statsd (%s for host replacement, must match nsqd)")
flagSet.Duration("statsd-interval", opts.StatsdInterval, "time interval nsqd is configured to push to statsd (must match nsqd)")
flagSet.String("notification-http-endpoint", "", "HTTP endpoint (fully qualified) to which POST notifications of admin actions will be sent")
flagSet.Duration("http-client-connect-timeout", opts.HTTPClientConnectTimeout, "timeout for HTTP connect")
flagSet.Duration("http-client-request-timeout", opts.HTTPClientRequestTimeout, "timeout for HTTP request")
flagSet.Bool("http-client-tls-insecure-skip-verify", false, "configure the HTTP client to skip verification of TLS certificates")
flagSet.String("http-client-tls-root-ca-file", "", "path to CA file for the HTTP client")
flagSet.String("http-client-tls-cert", "", "path to certificate file for the HTTP client")
flagSet.String("http-client-tls-key", "", "path to key file for the HTTP client")
flagSet.String("allow-config-from-cidr", opts.AllowConfigFromCIDR, "A CIDR from which to allow HTTP requests to the /config endpoint")
flagSet.String("acl-http-header", opts.ACLHTTPHeader, "HTTP header to check for authenticated admin users")
nsqlookupdHTTPAddresses := app.StringArray{}
flagSet.Var(&nsqlookupdHTTPAddresses, "lookupd-http-address", "lookupd HTTP address (may be given multiple times)")
nsqdHTTPAddresses := app.StringArray{}
flagSet.Var(&nsqdHTTPAddresses, "nsqd-http-address", "nsqd HTTP address (may be given multiple times)")
adminUsers := app.StringArray{}
flagSet.Var(&adminUsers, "admin-user", "admin user (may be given multiple times; if specified, only these users will be able to perform privileged actions; acl-http-header is used to determine the authenticated user)")
return flagSet
}
type program struct {
once sync.Once
nsqadmin *nsqadmin.NSQAdmin
}
func main() {
prg := &program{}
if err := svc.Run(prg, syscall.SIGINT, syscall.SIGTERM); err != nil {
logFatal("%s", err)
}
}
func (p *program) Init(env svc.Environment) error {
if env.IsWindowsService() {
dir := filepath.Dir(os.Args[0])
return os.Chdir(dir)
}
return nil
}
func (p *program) Start() error {
opts := nsqadmin.NewOptions()
flagSet := nsqadminFlagSet(opts)
flagSet.Parse(os.Args[1:])
if flagSet.Lookup("version").Value.(flag.Getter).Get().(bool) {
fmt.Println(version.String("nsqadmin"))
os.Exit(0)
}
var cfg config
configFile := flagSet.Lookup("config").Value.String()
if configFile != "" {
_, err := toml.DecodeFile(configFile, &cfg)
if err != nil {
logFatal("failed to load config file %s - %s", configFile, err)
}
}
cfg.Validate()
options.Resolve(opts, flagSet, cfg)
nsqadmin, err := nsqadmin.New(opts)
if err != nil {
logFatal("failed to instantiate nsqadmin - %s", err)
}
p.nsqadmin = nsqadmin
go func() {
err := p.nsqadmin.Main()
if err != nil {
p.Stop()
os.Exit(1)
}
}()
return nil
}
func (p *program) Stop() error {
p.once.Do(func() {
p.nsqadmin.Exit()
})
return nil
}
func logFatal(f string, args ...interface{}) {
lg.LogFatal("[nsqadmin] ", f, args...)
}
================================================
FILE: apps/nsqadmin/main_test.go
================================================
package main
import (
"testing"
"github.com/mreiferson/go-options"
"github.com/nsqio/nsq/internal/lg"
"github.com/nsqio/nsq/internal/test"
"github.com/nsqio/nsq/nsqadmin"
)
func TestConfigFlagParsing(t *testing.T) {
opts := nsqadmin.NewOptions()
opts.Logger = test.NewTestLogger(t)
flagSet := nsqadminFlagSet(opts)
flagSet.Parse([]string{})
cfg := config{"log_level": "debug"}
cfg.Validate()
options.Resolve(opts, flagSet, cfg)
if opts.LogLevel != lg.DEBUG {
t.Fatalf("log level: want debug, got %s", opts.LogLevel.String())
}
}
================================================
FILE: apps/nsqadmin/options.go
================================================
package main
import (
"fmt"
"github.com/nsqio/nsq/internal/lg"
)
type config map[string]interface{}
// Validate settings in the config file, and fatal on errors
func (cfg config) Validate() {
if v, exists := cfg["log_level"]; exists {
var t lg.LogLevel
err := t.Set(fmt.Sprintf("%v", v))
if err == nil {
cfg["log_level"] = t
} else {
logFatal("failed parsing log_level %+v", v)
}
}
}
================================================
FILE: apps/nsqd/README.md
================================================
## nsqd
`nsqd` is the daemon that receives, queues, and delivers messages to clients.
Read the [docs](https://nsq.io/components/nsqd.html).
================================================
FILE: apps/nsqd/main.go
================================================
package main
import (
"context"
"flag"
"fmt"
"math/rand"
"os"
"sync"
"syscall"
"time"
"github.com/BurntSushi/toml"
"github.com/judwhite/go-svc"
"github.com/mreiferson/go-options"
"github.com/nsqio/nsq/internal/lg"
"github.com/nsqio/nsq/internal/version"
"github.com/nsqio/nsq/nsqd"
)
type program struct {
once sync.Once
nsqd *nsqd.NSQD
}
func main() {
prg := &program{}
if err := svc.Run(prg, syscall.SIGINT, syscall.SIGTERM); err != nil {
logFatal("%s", err)
}
}
func (p *program) Init(env svc.Environment) error {
opts := nsqd.NewOptions()
flagSet := nsqdFlagSet(opts)
flagSet.Parse(os.Args[1:])
rand.Seed(time.Now().UTC().UnixNano())
if flagSet.Lookup("version").Value.(flag.Getter).Get().(bool) {
fmt.Println(version.String("nsqd"))
os.Exit(0)
}
var cfg config
configFile := flagSet.Lookup("config").Value.String()
if configFile != "" {
_, err := toml.DecodeFile(configFile, &cfg)
if err != nil {
logFatal("failed to load config file %s - %s", configFile, err)
}
}
cfg.Validate()
options.Resolve(opts, flagSet, cfg)
applyBackwardCompatibility(opts, flagSet)
nsqd, err := nsqd.New(opts)
if err != nil {
logFatal("failed to instantiate nsqd - %s", err)
}
p.nsqd = nsqd
return nil
}
func (p *program) Start() error {
err := p.nsqd.LoadMetadata()
if err != nil {
logFatal("failed to load metadata - %s", err)
}
err = p.nsqd.PersistMetadata()
if err != nil {
logFatal("failed to persist metadata - %s", err)
}
go func() {
err := p.nsqd.Main()
if err != nil {
p.Stop()
os.Exit(1)
}
}()
return nil
}
func (p *program) Stop() error {
p.once.Do(func() {
p.nsqd.Exit()
})
return nil
}
func (p *program) Handle(s os.Signal) error {
return svc.ErrStop
}
// Context returns a context that will be canceled when nsqd initiates the shutdown
func (p *program) Context() context.Context {
return p.nsqd.Context()
}
func logFatal(f string, args ...interface{}) {
lg.LogFatal("[nsqd] ", f, args...)
}
// applyBackwardCompatibility applies backward compatibility rules to options after flag resolution
func applyBackwardCompatibility(opts *nsqd.Options, flagSet *flag.FlagSet) {
// when max-defer-timeout was not explicitly set, refer to the max-req-timeout value
if flag := flagSet.Lookup("max-defer-timeout"); flag != nil && flag.Value.String() == flag.DefValue {
opts.MaxDeferTimeout = opts.MaxReqTimeout
}
// ... other backward compatibility rules can be added here
}
================================================
FILE: apps/nsqd/main_test.go
================================================
package main
import (
"crypto/tls"
"os"
"testing"
"github.com/BurntSushi/toml"
"github.com/mreiferson/go-options"
"github.com/nsqio/nsq/internal/lg"
"github.com/nsqio/nsq/internal/test"
"github.com/nsqio/nsq/nsqd"
)
func TestConfigFlagParsing(t *testing.T) {
opts := nsqd.NewOptions()
opts.Logger = test.NewTestLogger(t)
flagSet := nsqdFlagSet(opts)
flagSet.Parse([]string{})
var cfg config
f, err := os.Open("../../contrib/nsqd.cfg.example")
if err != nil {
t.Fatalf("%s", err)
}
defer f.Close()
toml.NewDecoder(f).Decode(&cfg)
cfg["log_level"] = "debug"
cfg.Validate()
options.Resolve(opts, flagSet, cfg)
nsqd.New(opts)
if opts.TLSMinVersion != tls.VersionTLS10 {
t.Errorf("min %#v not expected %#v", opts.TLSMinVersion, tls.VersionTLS10)
}
if opts.LogLevel != lg.DEBUG {
t.Fatalf("log level: want debug, got %s", opts.LogLevel.String())
}
}
================================================
FILE: apps/nsqd/options.go
================================================
package main
import (
"crypto/tls"
"flag"
"fmt"
"strconv"
"strings"
"github.com/nsqio/nsq/internal/app"
"github.com/nsqio/nsq/internal/lg"
"github.com/nsqio/nsq/nsqd"
)
type tlsRequiredOption int
func (t *tlsRequiredOption) Set(s string) error {
s = strings.ToLower(s)
if s == "tcp-https" {
*t = nsqd.TLSRequiredExceptHTTP
return nil
}
required, err := strconv.ParseBool(s)
if required {
*t = nsqd.TLSRequired
} else {
*t = nsqd.TLSNotRequired
}
return err
}
func (t *tlsRequiredOption) Get() interface{} { return int(*t) }
func (t *tlsRequiredOption) String() string {
return strconv.FormatInt(int64(*t), 10)
}
func (t *tlsRequiredOption) IsBoolFlag() bool { return true }
type tlsMinVersionOption uint16
var tlsVersionTable = []struct {
val uint16
str string
}{
{tls.VersionTLS10, "tls1.0"},
{tls.VersionTLS11, "tls1.1"},
{tls.VersionTLS12, "tls1.2"},
{tls.VersionTLS13, "tls1.3"},
}
func (t *tlsMinVersionOption) Set(s string) error {
s = strings.ToLower(s)
if s == "" {
return nil
}
for _, v := range tlsVersionTable {
if s == v.str {
*t = tlsMinVersionOption(v.val)
return nil
}
}
return fmt.Errorf("unknown tlsVersionOption %q", s)
}
func (t *tlsMinVersionOption) Get() interface{} { return uint16(*t) }
func (t *tlsMinVersionOption) String() string {
for _, v := range tlsVersionTable {
if uint16(*t) == v.val {
return v.str
}
}
return strconv.FormatInt(int64(*t), 10)
}
type config map[string]interface{}
// Validate settings in the config file, and fatal on errors
func (cfg config) Validate() {
// special validation/translation
if v, exists := cfg["tls_required"]; exists {
var t tlsRequiredOption
err := t.Set(fmt.Sprintf("%v", v))
if err == nil {
cfg["tls_required"] = t.String()
} else {
logFatal("failed parsing tls_required %+v", v)
}
}
if v, exists := cfg["tls_min_version"]; exists {
var t tlsMinVersionOption
err := t.Set(fmt.Sprintf("%v", v))
if err == nil {
newVal := fmt.Sprintf("%v", t.Get())
if newVal != "0" {
cfg["tls_min_version"] = newVal
} else {
delete(cfg, "tls_min_version")
}
} else {
logFatal("failed parsing tls_min_version %+v", v)
}
}
if v, exists := cfg["log_level"]; exists {
var t lg.LogLevel
err := t.Set(fmt.Sprintf("%v", v))
if err == nil {
cfg["log_level"] = t
} else {
logFatal("failed parsing log_level %+v", v)
}
}
}
func nsqdFlagSet(opts *nsqd.Options) *flag.FlagSet {
flagSet := flag.NewFlagSet("nsqd", flag.ExitOnError)
// basic options
flagSet.Bool("version", false, "print version string")
flagSet.String("config", "", "path to config file")
logLevel := opts.LogLevel
flagSet.Var(&logLevel, "log-level", "set log verbosity: debug, info, warn, error, or fatal")
flagSet.String("log-prefix", "[nsqd] ", "log message prefix")
flagSet.Bool("verbose", false, "[deprecated] has no effect, use --log-level")
flagSet.Int64("node-id", opts.ID, "unique part for message IDs, (int) in range [0,1024) (default is hash of hostname)")
flagSet.Bool("worker-id", false, "[deprecated] use --node-id")
flagSet.String("https-address", opts.HTTPSAddress, "<addr>:<port> to listen on for HTTPS clients")
flagSet.String("http-address", opts.HTTPAddress, "address to listen on for HTTP clients (<addr>:<port> for TCP/IP or <path> for unix socket)")
flagSet.String("tcp-address", opts.TCPAddress, "address to listen on for TCP clients (<addr>:<port> for TCP/IP or <path> for unix socket)")
authHTTPAddresses := app.StringArray{}
flagSet.Var(&authHTTPAddresses, "auth-http-address", "<addr>:<port> or a full url to query auth server (may be given multiple times)")
flagSet.String("auth-http-request-method", opts.AuthHTTPRequestMethod, "HTTP method to use for auth server requests")
flagSet.String("broadcast-address", opts.BroadcastAddress, "address that will be registered with lookupd (defaults to the OS hostname)")
flagSet.Int("broadcast-tcp-port", opts.BroadcastTCPPort, "TCP port that will be registered with lookupd (defaults to the TCP port that this nsqd is listening on)")
flagSet.Int("broadcast-http-port", opts.BroadcastHTTPPort, "HTTP port that will be registered with lookupd (defaults to the HTTP port that this nsqd is listening on)")
lookupdTCPAddrs := app.StringArray{}
flagSet.Var(&lookupdTCPAddrs, "lookupd-tcp-address", "lookupd TCP address (may be given multiple times)")
flagSet.Duration("http-client-connect-timeout", opts.HTTPClientConnectTimeout, "timeout for HTTP connect")
flagSet.Duration("http-client-request-timeout", opts.HTTPClientRequestTimeout, "timeout for HTTP request")
flagSet.String("topology-region", opts.TopologyRegion, "A region represents a larger domain, made up of one or more zones for preferring closer consumer")
flagSet.String("topology-zone", opts.TopologyZone, "A zone represents a logical failure domain for preferring closer consumer")
// diskqueue options
flagSet.String("data-path", opts.DataPath, "path to store disk-backed messages")
flagSet.Int64("mem-queue-size", opts.MemQueueSize, "number of messages to keep in memory (per topic/channel)")
flagSet.Int64("max-bytes-per-file", opts.MaxBytesPerFile, "number of bytes per diskqueue file before rolling")
flagSet.Int64("sync-every", opts.SyncEvery, "number of messages per diskqueue fsync")
flagSet.Duration("sync-timeout", opts.SyncTimeout, "duration of time per diskqueue fsync")
flagSet.Int("queue-scan-worker-pool-max", opts.QueueScanWorkerPoolMax, "max concurrency for checking in-flight and deferred message timeouts")
flagSet.Int("queue-scan-selection-count", opts.QueueScanSelectionCount, "number of channels to check per cycle (every 100ms) for in-flight and deferred timeouts")
// msg and command options
flagSet.Duration("msg-timeout", opts.MsgTimeout, "default duration to wait before auto-requeing a message")
flagSet.Duration("max-msg-timeout", opts.MaxMsgTimeout, "maximum duration before a message will timeout")
flagSet.Int64("max-msg-size", opts.MaxMsgSize, "maximum size of a single message in bytes")
flagSet.Duration("max-req-timeout", opts.MaxReqTimeout, "maximum requeuing timeout for a message")
flagSet.Int64("max-body-size", opts.MaxBodySize, "maximum size of a single command body")
flagSet.Duration("max-defer-timeout", opts.MaxDeferTimeout, "maximum duration when deferring a message")
// client overridable configuration options
flagSet.Duration("max-heartbeat-interval", opts.MaxHeartbeatInterval, "maximum client configurable duration of time between client heartbeats")
flagSet.Int64("max-rdy-count", opts.MaxRdyCount, "maximum RDY count for a client")
flagSet.Int64("max-output-buffer-size", opts.MaxOutputBufferSize, "maximum client configurable size (in bytes) for a client output buffer")
flagSet.Duration("max-output-buffer-timeout", opts.MaxOutputBufferTimeout, "maximum client configurable duration of time between flushing to a client")
flagSet.Duration("min-output-buffer-timeout", opts.MinOutputBufferTimeout, "minimum client configurable duration of time between flushing to a client")
flagSet.Duration("output-buffer-timeout", opts.OutputBufferTimeout, "default duration of time between flushing data to clients")
flagSet.Int("max-channel-consumers", opts.MaxChannelConsumers, "maximum channel consumer connection count per nsqd instance (default 0, i.e., unlimited)")
// statsd integration options
flagSet.String("statsd-address", opts.StatsdAddress, "UDP <addr>:<port> of a statsd daemon for pushing stats")
flagSet.Duration("statsd-interval", opts.StatsdInterval, "duration between pushing to statsd")
flagSet.Bool("statsd-mem-stats", opts.StatsdMemStats, "toggle sending memory and GC stats to statsd")
flagSet.String("statsd-prefix", opts.StatsdPrefix, "prefix used for keys sent to statsd (%s for host replacement)")
flagSet.Int("statsd-udp-packet-size", opts.StatsdUDPPacketSize, "the size in bytes of statsd UDP packets")
flagSet.Bool("statsd-exclude-ephemeral", opts.StatsdExcludeEphemeral, "Skip ephemeral topics and channels when sending stats to statsd")
// End to end percentile flags
e2eProcessingLatencyPercentiles := app.FloatArray{}
flagSet.Var(&e2eProcessingLatencyPercentiles, "e2e-processing-latency-percentile", "message processing time percentiles (as float (0, 1.0]) to track (can be specified multiple times or comma separated '1.0,0.99,0.95', default none)")
flagSet.Duration("e2e-processing-latency-window-time", opts.E2EProcessingLatencyWindowTime, "calculate end to end latency quantiles for this duration of time (ie: 60s would only show quantile calculations from the past 60 seconds)")
// TLS config
flagSet.String("tls-cert", opts.TLSCert, "path to certificate file")
flagSet.String("tls-key", opts.TLSKey, "path to key file")
flagSet.String("tls-client-auth-policy", opts.TLSClientAuthPolicy, "client certificate auth policy ('require' or 'require-verify')")
flagSet.String("tls-root-ca-file", opts.TLSRootCAFile, "path to certificate authority file")
tlsRequired := tlsRequiredOption(opts.TLSRequired)
tlsMinVersion := tlsMinVersionOption(opts.TLSMinVersion)
flagSet.Var(&tlsRequired, "tls-required", "require TLS for client connections (true, false, tcp-https)")
flagSet.Var(&tlsMinVersion, "tls-min-version", "minimum SSL/TLS version acceptable ('ssl3.0', 'tls1.0', 'tls1.1', 'tls1.2' or 'tls1.3')")
// compression
flagSet.Bool("deflate", opts.DeflateEnabled, "enable deflate feature negotiation (client compression)")
flagSet.Int("max-deflate-level", opts.MaxDeflateLevel, "max deflate compression level a client can negotiate (> values == > nsqd CPU usage)")
flagSet.Bool("snappy", opts.SnappyEnabled, "enable snappy feature negotiation (client compression)")
experiments := app.StringArray{}
var validExperiments []string
for _, e := range nsqd.AllExperiments {
validExperiments = append(validExperiments, fmt.Sprintf("%q", string(e)))
}
flagSet.Var(&experiments, "enable-experiment", fmt.Sprintf("enable experimental feature (may be given multiple times) (valid options: %s)", strings.Join(validExperiments, ", ")))
return flagSet
}
================================================
FILE: apps/nsqlookupd/README.md
================================================
## nsqlookupd
`nsqlookupd` is the daemon that manages topology metadata and serves client requests to
discover the location of topics at runtime.
Read the [docs](https://nsq.io/components/nsqlookupd.html).
================================================
FILE: apps/nsqlookupd/main.go
================================================
package main
import (
"flag"
"fmt"
"os"
"path/filepath"
"sync"
"syscall"
"github.com/BurntSushi/toml"
"github.com/judwhite/go-svc"
"github.com/mreiferson/go-options"
"github.com/nsqio/nsq/internal/lg"
"github.com/nsqio/nsq/internal/version"
"github.com/nsqio/nsq/nsqlookupd"
)
func nsqlookupdFlagSet(opts *nsqlookupd.Options) *flag.FlagSet {
flagSet := flag.NewFlagSet("nsqlookupd", flag.ExitOnError)
flagSet.String("config", "", "path to config file")
flagSet.Bool("version", false, "print version string")
logLevel := opts.LogLevel
flagSet.Var(&logLevel, "log-level", "set log verbosity: debug, info, warn, error, or fatal")
flagSet.String("log-prefix", "[nsqlookupd] ", "log message prefix")
flagSet.Bool("verbose", false, "[deprecated] has no effect, use --log-level")
flagSet.String("tcp-address", opts.TCPAddress, "<addr>:<port> to listen on for TCP clients")
flagSet.String("http-address", opts.HTTPAddress, "<addr>:<port> to listen on for HTTP clients")
flagSet.String("broadcast-address", opts.BroadcastAddress, "address of this lookupd node, (default to the OS hostname)")
flagSet.Duration("inactive-producer-timeout", opts.InactiveProducerTimeout, "duration of time a producer will remain in the active list since its last ping")
flagSet.Duration("tombstone-lifetime", opts.TombstoneLifetime, "duration of time a producer will remain tombstoned if registration remains")
return flagSet
}
type program struct {
once sync.Once
nsqlookupd *nsqlookupd.NSQLookupd
}
func main() {
prg := &program{}
if err := svc.Run(prg, syscall.SIGINT, syscall.SIGTERM); err != nil {
logFatal("%s", err)
}
}
func (p *program) Init(env svc.Environment) error {
if env.IsWindowsService() {
dir := filepath.Dir(os.Args[0])
return os.Chdir(dir)
}
return nil
}
func (p *program) Start() error {
opts := nsqlookupd.NewOptions()
flagSet := nsqlookupdFlagSet(opts)
flagSet.Parse(os.Args[1:])
if flagSet.Lookup("version").Value.(flag.Getter).Get().(bool) {
fmt.Println(version.String("nsqlookupd"))
os.Exit(0)
}
var cfg config
configFile := flagSet.Lookup("config").Value.String()
if configFile != "" {
_, err := toml.DecodeFile(configFile, &cfg)
if err != nil {
logFatal("failed to load config file %s - %s", configFile, err)
}
}
cfg.Validate()
options.Resolve(opts, flagSet, cfg)
nsqlookupd, err := nsqlookupd.New(opts)
if err != nil {
logFatal("failed to instantiate nsqlookupd", err)
}
p.nsqlookupd = nsqlookupd
go func() {
err := p.nsqlookupd.Main()
if err != nil {
p.Stop()
os.Exit(1)
}
}()
return nil
}
func (p *program) Stop() error {
p.once.Do(func() {
p.nsqlookupd.Exit()
})
return nil
}
func logFatal(f string, args ...interface{}) {
lg.LogFatal("[nsqlookupd] ", f, args...)
}
================================================
FILE: apps/nsqlookupd/main_test.go
================================================
package main
import (
"testing"
"github.com/mreiferson/go-options"
"github.com/nsqio/nsq/internal/lg"
"github.com/nsqio/nsq/internal/test"
"github.com/nsqio/nsq/nsqlookupd"
)
func TestConfigFlagParsing(t *testing.T) {
opts := nsqlookupd.NewOptions()
opts.Logger = test.NewTestLogger(t)
flagSet := nsqlookupdFlagSet(opts)
flagSet.Parse([]string{})
cfg := config{"log_level": "debug"}
cfg.Validate()
options.Resolve(opts, flagSet, cfg)
if opts.LogLevel != lg.DEBUG {
t.Fatalf("log level: want debug, got %s", opts.LogLevel.String())
}
}
================================================
FILE: apps/nsqlookupd/options.go
================================================
package main
import (
"fmt"
"github.com/nsqio/nsq/internal/lg"
)
type config map[string]interface{}
// Validate settings in the config file, and fatal on errors
func (cfg config) Validate() {
if v, exists := cfg["log_level"]; exists {
var t lg.LogLevel
err := t.Set(fmt.Sprintf("%v", v))
if err == nil {
cfg["log_level"] = t
} else {
logFatal("failed parsing log_level %+v", v)
}
}
}
================================================
FILE: apps/to_nsq/README.md
================================================
# to_nsq
A tool for publishing to an nsq topic with data from `stdin`.
## Usage
```
Usage of ./to_nsq:
-delimiter string
character to split input from stdin (default "\n")
-nsqd-tcp-address value
destination nsqd TCP address (may be given multiple times)
-producer-opt value
option to passthrough to nsq.Producer (may be given multiple times, http://godoc.org/github.com/nsqio/go-nsq#Config)
-rate int
Throttle messages to n/second. 0 to disable
-topic string
NSQ topic to publish to
```
### Examples
Publish each line of a file:
```bash
$ cat source.txt | to_nsq -topic="topic" -nsqd-tcp-address="127.0.0.1:4150"
```
Publish three messages, in one go:
```bash
$ echo "one,two,three" | to_nsq -delimiter="," -topic="topic" -nsqd-tcp-address="127.0.0.1:4150"
```
================================================
FILE: apps/to_nsq/to_nsq.go
================================================
// This is an NSQ client that publishes incoming messages from
// stdin to the specified topic.
package main
import (
"bufio"
"flag"
"fmt"
"io"
"log"
"os"
"os/signal"
"sync/atomic"
"syscall"
"time"
"github.com/nsqio/go-nsq"
"github.com/nsqio/nsq/internal/app"
"github.com/nsqio/nsq/internal/version"
)
var (
topic = flag.String("topic", "", "NSQ topic to publish to")
delimiter = flag.String("delimiter", "\n", "character to split input from stdin")
destNsqdTCPAddrs = app.StringArray{}
)
func init() {
flag.Var(&destNsqdTCPAddrs, "nsqd-tcp-address", "destination nsqd TCP address (may be given multiple times)")
}
func main() {
cfg := nsq.NewConfig()
flag.Var(&nsq.ConfigFlag{cfg}, "producer-opt", "option to passthrough to nsq.Producer (may be given multiple times, http://godoc.org/github.com/nsqio/go-nsq#Config)")
rate := flag.Int64("rate", 0, "Throttle messages to n/second. 0 to disable")
flag.Parse()
if len(*topic) == 0 {
log.Fatal("--topic required")
}
if len(*delimiter) != 1 {
log.Fatal("--delimiter must be a single byte")
}
stopChan := make(chan bool)
termChan := make(chan os.Signal, 1)
signal.Notify(termChan, syscall.SIGINT, syscall.SIGTERM)
cfg.UserAgent = fmt.Sprintf("to_nsq/%s go-nsq/%s", version.Binary, nsq.VERSION)
// make the producers
producers := make(map[string]*nsq.Producer)
for _, addr := range destNsqdTCPAddrs {
producer, err := nsq.NewProducer(addr, cfg)
if err != nil {
log.Fatalf("failed to create nsq.Producer - %s", err)
}
producers[addr] = producer
}
if len(producers) == 0 {
log.Fatal("--nsqd-tcp-address required")
}
throttleEnabled := *rate >= 1
balance := int64(1)
// avoid divide by 0 if !throttleEnabled
var interval time.Duration
if throttleEnabled {
interval = time.Second / time.Duration(*rate)
}
go func() {
if !throttleEnabled {
return
}
log.Printf("Throttling messages rate to max:%d/second", *rate)
// every tick increase the number of messages we can send
for range time.Tick(interval) {
n := atomic.AddInt64(&balance, 1)
// if we build up more than 1s of capacity just bound to that
if n > int64(*rate) {
atomic.StoreInt64(&balance, int64(*rate))
}
}
}()
r := bufio.NewReader(os.Stdin)
delim := (*delimiter)[0]
go func() {
for {
var err error
if throttleEnabled {
currentBalance := atomic.LoadInt64(&balance)
if currentBalance <= 0 {
time.Sleep(interval)
}
err = readAndPublish(r, delim, producers)
atomic.AddInt64(&balance, -1)
} else {
err = readAndPublish(r, delim, producers)
}
if err != nil {
if err != io.EOF {
log.Fatal(err)
}
close(stopChan)
break
}
}
}()
select {
case <-termChan:
case <-stopChan:
}
for _, producer := range producers {
producer.Stop()
}
}
// readAndPublish reads to the delim from r and publishes the bytes
// to the map of producers.
func readAndPublish(r *bufio.Reader, delim byte, producers map[string]*nsq.Producer) error {
line, readErr := r.ReadBytes(delim)
if len(line) > 0 {
// trim the delimiter
line = line[:len(line)-1]
}
if len(line) == 0 {
return readErr
}
for _, producer := range producers {
err := producer.Publish(*topic, line)
if err != nil {
return err
}
}
return readErr
}
================================================
FILE: bench/bench.py
================================================
#!/usr/bin/env python3
#
# This script bootstraps an NSQ cluster in EC2 and runs benchmarks.
#
# Requires python3 and the following packages:
# - boto3
# - paramiko
# - tornado
#
# AWS authentication is delegated entirely to the boto3 environment, see:
#
# https://boto3.amazonaws.com/v1/documentation/api/latest/guide/configuration.html
#
# EC2 instances are launched into EC2 Classic, expecting a 'default' security group
# that allows allows SSH (port 22) from 0.0.0.0/0 and an EC2 key pair created
# (named 'default', but configurable via --ssh-key-name).
#
import sys
import logging
import time
import datetime
import socket
import warnings
import hashlib
import boto3
import paramiko.client
import paramiko.ssh_exception
import tornado.options
def ssh_connect_with_retries(host, retries=3, timeout=30):
for i in range(retries):
try:
ssh_client = paramiko.client.SSHClient()
ssh_client.set_missing_host_key_policy(paramiko.client.WarningPolicy())
ssh_client.connect(host, username='ubuntu', timeout=timeout)
return ssh_client
except (socket.error, paramiko.ssh_exception.SSHException):
if i == retries - 1:
raise
logging.warning('... re-trying to connect to %s:%d in 15s', host, 22)
time.sleep(15)
def ssh_cmd_async(ssh_client, cmd):
transport = ssh_client.get_transport()
chan = transport.open_session()
chan.exec_command(cmd)
return chan
def ssh_cmd(ssh_client, cmd, timeout=2):
transport = ssh_client.get_transport()
chan = transport.open_session()
chan.settimeout(timeout)
chan.exec_command(cmd)
stdout = b''
stderr = b''
while True:
if chan.recv_ready():
stdout += chan.recv(4096)
continue
if chan.recv_stderr_ready():
stderr += chan.recv_stderr(4096)
continue
if chan.exit_status_ready():
exit_status = chan.recv_exit_status()
break
time.sleep(0.1)
if exit_status != 0:
raise Exception('%r' % stderr)
return stdout, stderr
def get_session():
return boto3.session.Session(region_name=tornado.options.options.region)
def _bootstrap(addr):
commit = tornado.options.options.commit
golang_version = tornado.options.options.golang_version
ssh_client = ssh_connect_with_retries(addr)
for cmd in [
'wget https://storage.googleapis.com/golang/go%s.linux-amd64.tar.gz' % golang_version,
'sudo -S tar -C /usr/local -xzf go%s.linux-amd64.tar.gz' % golang_version,
'sudo -S apt-get update',
'sudo -S apt-get -y install git mercurial',
'mkdir -p go/src/github.com/nsqio',
'cd go/src/github.com/nsqio && git clone https://github.com/nsqio/nsq',
'cd go/src/github.com/nsqio/nsq && git checkout %s' % commit,
'cd go/src/github.com/nsqio/nsq/apps/nsqd && GO111MODULE=on /usr/local/go/bin/go build',
'cd go/src/github.com/nsqio/nsq/bench/bench_writer && GO111MODULE=on /usr/local/go/bin/go build',
'cd go/src/github.com/nsqio/nsq/bench/bench_reader && GO111MODULE=on /usr/local/go/bin/go build',
'sudo -S mkdir -p /mnt/nsq',
'sudo -S chmod 777 /mnt/nsq']:
ssh_cmd(ssh_client, cmd, timeout=10)
def bootstrap():
session = get_session()
ec2 = session.resource('ec2')
total_count = tornado.options.options.nsqd_count + tornado.options.options.worker_count
logging.info('launching %d instances', total_count)
instances = ec2.create_instances(
ImageId=tornado.options.options.ami,
MinCount=total_count,
MaxCount=total_count,
KeyName=tornado.options.options.ssh_key_name,
InstanceType=tornado.options.options.instance_type,
SecurityGroups=['default'])
logging.info('waiting for instances to launch...')
while any(i.state['Name'] != 'running' for i in instances):
waiting_for = [i.id for i in instances if i.state['Name'] != 'running']
logging.info('... sleeping for 5s (waiting for %s)', ', '.join(waiting_for))
time.sleep(5)
for instance in instances:
instance.load()
for instance in instances:
if not instance.tags:
instance.create_tags(Tags=[{'Key': 'nsq_bench', 'Value': '1'}])
try:
c = 0
for i in instances:
c += 1
logging.info('(%d) bootstrapping %s (%s)', c, i.public_dns_name, i.id)
_bootstrap(i.public_dns_name)
except Exception:
logging.exception('bootstrap failed')
decomm()
def run():
instances = _find_instances()
logging.info('launching nsqd on %d host(s)', tornado.options.options.nsqd_count)
nsqd_chans = []
nsqd_hosts = instances[:tornado.options.options.nsqd_count]
for instance in nsqd_hosts:
try:
ssh_client = ssh_connect_with_retries(instance.public_dns_name)
for cmd in [
'sudo -S pkill -f nsqd',
'sudo -S rm -f /mnt/nsq/*.dat',
'GOMAXPROCS=32 ./go/src/github.com/nsqio/nsq/apps/nsqd/nsqd \
--data-path=/mnt/nsq \
--mem-queue-size=10000000 \
--max-rdy-count=%s' % (tornado.options.options.rdy)]:
nsqd_chans.append((ssh_client, ssh_cmd_async(ssh_client, cmd)))
except Exception:
logging.exception('failed')
nsqd_tcp_addrs = [i.public_dns_name for i in nsqd_hosts]
dt = datetime.datetime.utcnow()
deadline = dt + datetime.timedelta(seconds=30)
logging.info('launching %d producer(s) on %d host(s)',
tornado.options.options.nsqd_count * tornado.options.options.worker_count,
tornado.options.options.worker_count)
worker_chans = []
producer_hosts = instances[tornado.options.options.nsqd_count:]
for instance in producer_hosts:
for nsqd_tcp_addr in nsqd_tcp_addrs:
topic = hashlib.md5(instance.public_dns_name.encode('utf-8')).hexdigest()
try:
ssh_client = ssh_connect_with_retries(instance.public_dns_name)
for cmd in [
'GOMAXPROCS=2 \
./go/src/github.com/nsqio/nsq/bench/bench_writer/bench_writer \
--topic=%s --nsqd-tcp-address=%s:4150 --deadline=\'%s\' --size=%d' % (
topic, nsqd_tcp_addr, deadline.strftime('%Y-%m-%d %H:%M:%S'),
tornado.options.options.msg_size)]:
worker_chans.append((ssh_client, ssh_cmd_async(ssh_client, cmd)))
except Exception:
logging.exception('failed')
if tornado.options.options.mode == 'pubsub':
logging.info('launching %d consumer(s) on %d host(s)',
tornado.options.options.nsqd_count * tornado.options.options.worker_count,
tornado.options.options.worker_count)
consumer_hosts = instances[tornado.options.options.nsqd_count:]
for instance in consumer_hosts:
for nsqd_tcp_addr in nsqd_tcp_addrs:
topic = hashlib.md5(instance.public_dns_name.encode('utf-8')).hexdigest()
try:
ssh_client = ssh_connect_with_retries(instance.public_dns_name)
for cmd in [
'GOMAXPROCS=8 \
./go/src/github.com/nsqio/nsq/bench/bench_reader/bench_reader \
--topic=%s --nsqd-tcp-address=%s:4150 --deadline=\'%s\' --size=%d \
--rdy=%d' % (
topic, nsqd_tcp_addr, deadline.strftime('%Y-%m-%d %H:%M:%S'),
tornado.options.options.msg_size, tornado.options.options.rdy)]:
worker_chans.append((ssh_client, ssh_cmd_async(ssh_client, cmd)))
except Exception:
logging.exception('failed')
stats = {
'bench_reader': {
'durations': [],
'mbytes': [],
'ops': []
},
'bench_writer': {
'durations': [],
'mbytes': [],
'ops': []
}
}
while worker_chans:
for ssh_client, chan in worker_chans[:]:
if chan.recv_ready():
sys.stdout.write(chan.recv(4096))
sys.stdout.flush()
continue
if chan.recv_stderr_ready():
line = chan.recv_stderr(4096).decode('utf-8')
if 'duration:' in line:
kind = line.split(' ')[0][1:-1]
parts = line.rsplit('duration:')[1].split('-')
stats[kind]['durations'].append(float(parts[0].strip()[:-1]))
stats[kind]['mbytes'].append(float(parts[1].strip()[:-4]))
stats[kind]['ops'].append(float(parts[2].strip()[:-5]))
sys.stdout.write(line)
sys.stdout.flush()
continue
if chan.exit_status_ready():
worker_chans.remove((ssh_client, chan))
time.sleep(0.1)
for kind, data in stats.items():
if not data['durations']:
continue
max_duration = max(data['durations'])
total_mb = sum(data['mbytes'])
total_ops = sum(data['ops'])
logging.info('[%s] %fs - %fmb/s - %fops/s - %fus/op',
kind, max_duration, total_mb, total_ops,
max_duration / total_ops * 1000 * 1000)
for ssh_client, chan in nsqd_chans:
chan.close()
def _find_instances():
session = get_session()
ec2 = session.resource('ec2')
return [i for i in ec2.instances.all() if
i.state['Name'] == 'running' and any(t['Key'] == 'nsq_bench' for t in i.tags)]
def decomm():
instances = _find_instances()
logging.info('terminating instances %s' % ','.join(i.id for i in instances))
for instance in instances:
instance.terminate()
if __name__ == '__main__':
tornado.options.define('region', type=str, default='us-east-1',
help='EC2 region to launch instances')
tornado.options.define('nsqd_count', type=int, default=3,
help='how many nsqd instances to launch')
tornado.options.define('worker_count', type=int, default=3,
help='how many worker instances to launch')
# ubuntu 18.04 HVM instance store us-east-1
tornado.options.define('ami', type=str, default='ami-0938f2289b3fa3f5b',
help='AMI ID')
tornado.options.define('ssh_key_name', type=str, default='default',
help='SSH key name')
tornado.options.define('instance_type', type=str, default='c3.2xlarge',
help='EC2 instance type')
tornado.options.define('msg_size', type=int, default=200,
help='size of message')
tornado.options.define('rdy', type=int, default=10000,
help='RDY count to use for bench_reader')
tornado.options.define('mode', type=str, default='pubsub',
help='the benchmark mode (pub, pubsub)')
tornado.options.define('commit', type=str, default='master',
help='the git commit')
tornado.options.define('golang_version', type=str, default='1.14.3',
help='the go version')
tornado.options.parse_command_line()
logging.getLogger('paramiko').setLevel(logging.WARNING)
warnings.simplefilter('ignore')
cmd_name = sys.argv[-1]
cmd_map = {
'bootstrap': bootstrap,
'run': run,
'decomm': decomm
}
cmd = cmd_map.get(cmd_name, bootstrap)
sys.exit(cmd())
================================================
FILE: bench/bench_channels/bench_channels.go
================================================
package main
import (
"bufio"
"flag"
"fmt"
"net"
"sync"
"time"
"github.com/nsqio/go-nsq"
)
var (
num = flag.Int("num", 10000, "num channels")
tcpAddress = flag.String("nsqd-tcp-address", "127.0.0.1:4150", "<addr>:<port> to connect to nsqd")
)
func main() {
flag.Parse()
var wg sync.WaitGroup
goChan := make(chan int)
rdyChan := make(chan int)
for j := 0; j < *num; j++ {
wg.Add(1)
go func(id int) {
subWorker(*num, *tcpAddress, fmt.Sprintf("t%d", j), "ch", rdyChan, goChan, id)
wg.Done()
}(j)
<-rdyChan
time.Sleep(5 * time.Millisecond)
}
close(goChan)
wg.Wait()
}
func subWorker(n int, tcpAddr string,
topic string, channel string,
rdyChan chan int, goChan chan int, id int) {
conn, err := net.DialTimeout("tcp", tcpAddr, time.Second)
if err != nil {
panic(err.Error())
}
conn.Write(nsq.MagicV2)
rw := bufio.NewReadWriter(bufio.NewReader(conn), bufio.NewWriter(conn))
ci := make(map[string]interface{})
ci["client_id"] = "test"
cmd, _ := nsq.Identify(ci)
cmd.WriteTo(rw)
nsq.Subscribe(topic, channel).WriteTo(rw)
rdyCount := 1
rdy := rdyCount
rdyChan <- 1
<-goChan
nsq.Ready(rdyCount).WriteTo(rw)
rw.Flush()
nsq.ReadResponse(rw)
nsq.ReadResponse(rw)
for {
resp, err := nsq.ReadResponse(rw)
if err != nil {
panic(err.Error())
}
frameType, data, err := nsq.UnpackResponse(resp)
if err != nil {
panic(err.Error())
}
if frameType == nsq.FrameTypeError {
panic(string(data))
} else if frameType == nsq.FrameTypeResponse {
nsq.Nop().WriteTo(rw)
rw.Flush()
continue
}
msg, err := nsq.DecodeMessage(data)
if err != nil {
panic(err.Error())
}
nsq.Finish(msg.ID).WriteTo(rw)
rdy--
if rdy == 0 {
nsq.Ready(rdyCount).WriteTo(rw)
rdy = rdyCount
rw.Flush()
}
}
}
================================================
FILE: bench/bench_reader/bench_reader.go
================================================
package main
import (
"bufio"
"errors"
"flag"
"fmt"
"log"
"net"
"runtime"
"sync"
"sync/atomic"
"time"
"github.com/nsqio/go-nsq"
)
var (
runfor = flag.Duration("runfor", 10*time.Second, "duration of time to run")
tcpAddress = flag.String("nsqd-tcp-address", "127.0.0.1:4150", "<addr>:<port> to connect to nsqd")
size = flag.Int("size", 200, "size of messages")
topic = flag.String("topic", "sub_bench", "topic to receive messages on")
channel = flag.String("channel", "ch", "channel to receive messages on")
deadline = flag.String("deadline", "", "deadline to start the benchmark run")
rdy = flag.Int("rdy", 2500, "RDY count to use")
)
var totalMsgCount int64
func main() {
flag.Parse()
var wg sync.WaitGroup
log.SetPrefix("[bench_reader] ")
goChan := make(chan int)
rdyChan := make(chan int)
workers := runtime.GOMAXPROCS(0)
for j := 0; j < workers; j++ {
wg.Add(1)
go func(id int) {
subWorker(*runfor, workers, *tcpAddress, *topic, *channel, rdyChan, goChan, id)
wg.Done()
}(j)
<-rdyChan
}
if *deadline != "" {
t, err := time.Parse("2006-01-02 15:04:05", *deadline)
if err != nil {
log.Fatal(err)
}
d := time.Until(t)
log.Printf("sleeping until %s (%s)", t, d)
time.Sleep(d)
}
start := time.Now()
close(goChan)
wg.Wait()
end := time.Now()
duration := end.Sub(start)
tmc := atomic.LoadInt64(&totalMsgCount)
log.Printf("duration: %s - %.03fmb/s - %.03fops/s - %.03fus/op",
duration,
float64(tmc*int64(*size))/duration.Seconds()/1024/1024,
float64(tmc)/duration.Seconds(),
float64(duration/time.Microsecond)/float64(tmc))
}
func subWorker(td time.Duration, workers int, tcpAddr string, topic string, channel string, rdyChan chan int, goChan chan int, id int) {
conn, err := net.DialTimeout("tcp", tcpAddr, time.Second)
if err != nil {
panic(err.Error())
}
conn.Write(nsq.MagicV2)
rw := bufio.NewReadWriter(bufio.NewReader(conn), bufio.NewWriter(conn))
ci := make(map[string]interface{})
ci["client_id"] = "reader"
ci["hostname"] = "reader"
ci["user_agent"] = fmt.Sprintf("bench_reader/%s", nsq.VERSION)
cmd, _ := nsq.Identify(ci)
cmd.WriteTo(rw)
nsq.Subscribe(topic, channel).WriteTo(rw)
rdyChan <- 1
<-goChan
nsq.Ready(*rdy).WriteTo(rw)
rw.Flush()
nsq.ReadResponse(rw)
nsq.ReadResponse(rw)
var msgCount int64
go func() {
time.Sleep(td)
conn.Close()
}()
for {
resp, err := nsq.ReadResponse(rw)
if err != nil {
if errors.Is(err, net.ErrClosed) {
break
}
panic(err.Error())
}
frameType, data, err := nsq.UnpackResponse(resp)
if err != nil {
panic(err.Error())
}
if frameType == nsq.FrameTypeError {
panic(string(data))
} else if frameType == nsq.FrameTypeResponse {
continue
}
msg, err := nsq.DecodeMessage(data)
if err != nil {
panic(err.Error())
}
nsq.Finish(msg.ID).WriteTo(rw)
msgCount++
if float64(msgCount%int64(*rdy)) > float64(*rdy)*0.75 {
rw.Flush()
}
}
atomic.AddInt64(&totalMsgCount, msgCount)
}
================================================
FILE: bench/bench_writer/bench_writer.go
================================================
package main
import (
"bufio"
"flag"
"fmt"
"log"
"net"
"runtime"
"sync"
"sync/atomic"
"time"
"github.com/nsqio/go-nsq"
)
var (
runfor = flag.Duration("runfor", 10*time.Second, "duration of time to run")
tcpAddress = flag.String("nsqd-tcp-address", "127.0.0.1:4150", "<addr>:<port> to connect to nsqd")
topic = flag.String("topic", "sub_bench", "topic to receive messages on")
size = flag.Int("size", 200, "size of messages")
batchSize = flag.Int("batch-size", 200, "batch size of messages")
deadline = flag.String("deadline", "", "deadline to start the benchmark run")
)
var totalMsgCount int64
func main() {
flag.Parse()
var wg sync.WaitGroup
log.SetPrefix("[bench_writer] ")
msg := make([]byte, *size)
batch := make([][]byte, *batchSize)
for i := range batch {
batch[i] = msg
}
goChan := make(chan int)
rdyChan := make(chan int)
for j := 0; j < runtime.GOMAXPROCS(0); j++ {
wg.Add(1)
go func() {
pubWorker(*runfor, *tcpAddress, *batchSize, batch, *topic, rdyChan, goChan)
wg.Done()
}()
<-rdyChan
}
if *deadline != "" {
t, err := time.Parse("2006-01-02 15:04:05", *deadline)
if err != nil {
log.Fatal(err)
}
d := time.Until(t)
log.Printf("sleeping until %s (%s)", t, d)
time.Sleep(d)
}
start := time.Now()
close(goChan)
wg.Wait()
end := time.Now()
duration := end.Sub(start)
tmc := atomic.LoadInt64(&totalMsgCount)
log.Printf("duration: %s - %.03fmb/s - %.03fops/s - %.03fus/op",
duration,
float64(tmc*int64(*size))/duration.Seconds()/1024/1024,
float64(tmc)/duration.Seconds(),
float64(duration/time.Microsecond)/float64(tmc))
}
func pubWorker(td time.Duration, tcpAddr string, batchSize int, batch [][]byte, topic string, rdyChan chan int, goChan chan int) {
conn, err := net.DialTimeout("tcp", tcpAddr, time.Second)
if err != nil {
panic(err.Error())
}
conn.Write(nsq.MagicV2)
rw := bufio.NewReadWriter(bufio.NewReader(conn), bufio.NewWriter(conn))
ci := make(map[string]interface{})
ci["client_id"] = "writer"
ci["hostname"] = "writer"
ci["user_agent"] = fmt.Sprintf("bench_writer/%s", nsq.VERSION)
cmd, _ := nsq.Identify(ci)
cmd.WriteTo(rw)
rdyChan <- 1
<-goChan
rw.Flush()
nsq.ReadResponse(rw)
var msgCount int64
endTime := time.Now().Add(td)
for {
cmd, _ := nsq.MultiPublish(topic, batch)
_, err := cmd.WriteTo(rw)
if err != nil {
panic(err.Error())
}
err = rw.Flush()
if err != nil {
panic(err.Error())
}
resp, err := nsq.ReadResponse(rw)
if err != nil {
panic(err.Error())
}
frameType, data, err := nsq.UnpackResponse(resp)
if err != nil {
panic(err.Error())
}
if frameType == nsq.FrameTypeError {
panic(string(data))
}
msgCount += int64(len(batch))
if time.Now().After(endTime) {
break
}
}
atomic.AddInt64(&totalMsgCount, msgCount)
}
================================================
FILE: bench/requirements.txt
================================================
tornado==4.3
paramiko==1.16.0
boto==2.39.0
================================================
FILE: bench.sh
================================================
#!/bin/bash
readonly messageSize="${1:-200}"
readonly batchSize="${2:-200}"
readonly memQueueSize="${3:-1000000}"
readonly dataPath="${4:-}"
set -e
set -u
echo "# using --mem-queue-size=$memQueueSize --data-path=$dataPath --size=$messageSize --batch-size=$batchSize"
echo "# compiling/running nsqd"
pushd apps/nsqd >/dev/null
go build
rm -f *.dat
./nsqd --mem-queue-size=$memQueueSize --data-path=$dataPath >/dev/null 2>&1 &
nsqd_pid=$!
popd >/dev/null
cleanup() {
kill -9 $nsqd_pid
rm -f nsqd/*.dat
}
trap cleanup INT TERM EXIT
sleep 0.3
echo "# creating topic/channel"
curl --silent 'http://127.0.0.1:4151/create_topic?topic=sub_bench' >/dev/null 2>&1
curl --silent 'http://127.0.0.1:4151/create_channel?topic=sub_bench&channel=ch' >/dev/null 2>&1
echo "# compiling bench_reader/bench_writer"
pushd bench >/dev/null
for app in bench_reader bench_writer; do
pushd $app >/dev/null
go build
popd >/dev/null
done
popd >/dev/null
echo -n "PUB: "
bench/bench_writer/bench_writer --size=$messageSize --batch-size=$batchSize 2>&1
curl -s -o cpu.pprof http://127.0.0.1:4151/debug/pprof/profile &
pprof_pid=$!
echo -n "SUB: "
bench/bench_reader/bench_reader --size=$messageSize --channel=ch 2>&1
echo "waiting for pprof..."
wait $pprof_pid
================================================
FILE: contrib/nsq.spec
================================================
%define name nsq
%define version 1.1.1-alpha
%define release 1
%define path usr/local
%define group Database/Applications
%define __os_install_post %{nil}
Summary: nsq
Name: %{name}
Version: %{version}
Release: %{release}
Group: %{group}
Packager: Matt Reiferson <mreiferson@gmail.com>
License: Apache
BuildRoot: %{_tmppath}/%{name}-%{version}-%{release}
AutoReqProv: no
# we just assume you have go installed. You may or may not have an RPM to depend on.
# BuildRequires: go
%description
NSQ - A realtime distributed messaging platform
https://github.com/nsqio/nsq
%prep
mkdir -p $RPM_BUILD_DIR/%{name}-%{version}-%{release}
cd $RPM_BUILD_DIR/%{name}-%{version}-%{release}
git clone git@github.com:nsqio/nsq.git
%build
cd $RPM_BUILD_DIR/%{name}-%{version}-%{release}/nsq
make PREFIX=/%{path}
%install
export DONT_STRIP=1
rm -rf $RPM_BUILD_ROOT
cd $RPM_BUILD_DIR/%{name}-%{version}-%{release}/nsq
make PREFIX=/${path} DESTDIR=$RPM_BUILD_ROOT install
%files
/%{path}/bin/nsqadmin
/%{path}/bin/nsqd
/%{path}/bin/nsqlookupd
/%{path}/bin/nsq_to_file
/%{path}/bin/nsq_to_http
/%{path}/bin/nsq_to_nsq
/%{path}/bin/nsq_tail
/%{path}/bin/nsq_stat
/%{path}/bin/to_nsq
================================================
FILE: contrib/nsqadmin.cfg.example
================================================
## log verbosity level: debug, info, warn, error, or fatal
log_level = "info"
## log message prefix (default "[nsqadmin] ")
# log_prefix = ""
## HTTP header to check for authenticated admin users (default "X_Forwarded_User")
# acl_http_header = ""
## admin user (may be given multiple times; if specified, only these users will be able to perform privileged actions)
# admin_users = [
# "admin"
# ]
## A CIDR from which to allow HTTP requests to the /config endpoint (default "127.0.0.1/8")
# allow_config_from_cidr = ""
## URL base path (default "/")
# base_path = ""
## timeout for HTTP connect (default 2s)
# http_client_connect_timeout = "2s"
## timeout for HTTP request (default 5s)
# http_client_request_timeout = "5s"
## path to certificate file for the HTTP client
# http_client_tls_cert = ""
## configure the HTTP client to skip verification of TLS certificates
# http_client_tls_insecure_skip_verify = false
## path to key file for the HTTP client
# http_client_tls_key = ""
## path to CA file for the HTTP client
# http_client_tls_root_ca_file = ""
## <addr>:<port> to listen on for HTTP clients
http_address = "0.0.0.0:4171"
## graphite HTTP address
graphite_url = ""
## proxy HTTP requests to graphite
proxy_graphite = false
## prefix used for keys sent to statsd (%s for host replacement, must match nsqd)
statsd_prefix = "nsq.%s"
## format of statsd counter stats
statsd_counter_format = "stats.counters.%s.count"
## format of statsd gauge stats
statsd_gauge_format = "stats.gauges.%s"
## time interval nsqd is configured to push to statsd (must match nsqd)
statsd_interval = "60s"
## HTTP endpoint (fully qualified) to which POST notifications of admin actions will be sent
notification_http_endpoint = ""
## nsqlookupd HTTP addresses
nsqlookupd_http_addresses = [
"127.0.0.1:4161"
]
## nsqd HTTP addresses (optional)
nsqd_http_addresses = [
"127.0.0.1:4151"
]
================================================
FILE: contrib/nsqd.cfg.example
================================================
## log verbosity level: debug, info, warn, error, or fatal
log_level = "info"
## unique identifier (int) for this worker (will default to a hash of hostname)
# id = 5150
## <addr>:<port> to listen on for TCP clients
tcp_address = "0.0.0.0:4150"
## <addr>:<port> to listen on for HTTP clients
http_address = "0.0.0.0:4151"
## <addr>:<port> to listen on for HTTPS clients
# https_address = "0.0.0.0:4152"
## address that will be registered with lookupd (defaults to the OS hostname)
# broadcast_address = ""
## cluster of nsqlookupd TCP addresses
nsqlookupd_tcp_addresses = [
"127.0.0.1:4160"
]
## duration to wait before HTTP client connection timeout
http_client_connect_timeout = "2s"
## duration to wait before HTTP client request timeout
http_client_request_timeout = "5s"
## path to store disk-backed messages
# data_path = "/var/lib/nsq"
## number of messages to keep in memory (per topic/channel)
mem_queue_size = 10000
## number of bytes per diskqueue file before rolling
max_bytes_per_file = 104857600
## number of messages per diskqueue fsync
sync_every = 2500
## duration of time per diskqueue fsync (time.Duration)
sync_timeout = "2s"
## duration to wait before auto-requeing a message
msg_timeout = "60s"
## maximum duration before a message will timeout
max_msg_timeout = "15m"
## maximum size of a single message in bytes
max_msg_size = 1024768
## maximum requeuing timeout for a message
max_req_timeout = "1h"
## maximum size of a single command body
max_body_size = 5123840
## maximum client configurable duration of time between client heartbeats
max_heartbeat_interval = "60s"
## maximum RDY count for a client
max_rdy_count = 2500
## maximum client configurable size (in bytes) for a client output buffer
max_output_buffer_size = 65536
## maximum client configurable duration of time between flushing to a client (time.Duration)
max_output_buffer_timeout = "1s"
## UDP <addr>:<port> of a statsd daemon for pushing stats
# statsd_address = "127.0.0.1:8125"
## prefix used for keys sent to statsd (%s for host replacement)
statsd_prefix = "nsq.%s"
## duration between pushing to statsd (time.Duration)
statsd_interval = "60s"
## toggle sending memory and GC stats to statsd
statsd_mem_stats = true
## the size in bytes of statsd UDP packets
# statsd_udp_packet_size = 508
## message processing time percentiles to keep track of (float)
e2e_processing_latency_percentiles = [
1.0,
0.99,
0.95
]
## calculate end to end latency quantiles for this duration of time (time.Duration)
e2e_processing_latency_window_time = "10m"
## path to certificate file
tls_cert = ""
## path to private key file
tls_key = ""
## set policy on client certificate (require - client must provide certificate,
## require-verify - client must provide verifiable signed certificate)
# tls_client_auth_policy = "require-verify"
## set custom root Certificate Authority
# tls_root_ca_file = ""
## require client TLS upgrades
tls_required = false
## minimum TLS version ("ssl3.0", "tls1.0," "tls1.1", "tls1.2")
tls_min_version = ""
## enable deflate feature negotiation (client compression)
deflate = true
## max deflate compression level a client can negotiate (> values == > nsqd CPU usage)
max_deflate_level = 6
## enable snappy feature negotiation (client compression)
snappy = true
================================================
FILE: contrib/nsqlookupd.cfg.example
================================================
## log verbosity level: debug, info, warn, error, or fatal
log_level = "info"
## <addr>:<port> to listen on for TCP clients
tcp_address = "0.0.0.0:4160"
## <addr>:<port> to listen on for HTTP clients
http_address = "0.0.0.0:4161"
## address that will be registered with lookupd (defaults to the OS hostname)
# broadcast_address = ""
## duration of time a producer will remain in the active list since its last ping
inactive_producer_timeout = "300s"
## duration of time a producer will remain tombstoned if registration remains
tombstone_lifetime = "45s"
================================================
FILE: coverage.sh
================================================
#!/bin/bash
# Generate test coverage statistics for Go packages.
#
# Works around the fact that `go test -coverprofile` currently does not work
# with multiple packages, see https://code.google.com/p/go/issues/detail?id=6909
#
# Usage: coverage.sh [--html|--coveralls]
#
# --html Additionally create HTML report
# --coveralls Push coverage statistics to coveralls.io
#
set -e
workdir=.cover
profile="$workdir/cover.out"
mode=count
generate_cover_data() {
rm -rf "$workdir"
mkdir "$workdir"
for pkg in "$@"; do
f="$workdir/$(echo $pkg | tr / -).cover"
go test -covermode="$mode" -coverprofile="$f" "$pkg"
done
echo "mode: $mode" >"$profile"
grep -h -v "^mode:" "$workdir"/*.cover >>"$profile"
}
show_html_report() {
go tool cover -html="$profile" -o="$workdir"/coverage.html
}
show_csv_report() {
go tool cover -func="$profile" -o="$workdir"/coverage.csv
}
push_to_coveralls() {
echo "Pushing coverage statistics to coveralls.io"
# ignore failure to push - it happens
$GOPATH/bin/goveralls -coverprofile="$profile" \
-service=github \
-ignore="nsqadmin/bindata.go" || true
}
generate_cover_data $(go list ./... | grep -v /vendor/)
show_csv_report
case "$1" in
"")
;;
--html)
show_html_report ;;
--coveralls)
push_to_coveralls ;;
*)
echo >&2 "error: invalid option: $1"; exit 1 ;;
esac
================================================
FILE: dist.sh
================================================
#!/bin/bash
# 1. commit to bump the version and update the changelog/readme
# 2. tag that commit
# 3. use dist.sh to produce tar.gz for all platforms
# 4. aws s3 cp dist s3://bitly-downloads/nsq/ --recursive --exclude "*" --include "nsq-1.3.0*" --profile bitly --acl public-read
# 5. docker manifest push nsqio/nsq:latest
# 6. push to nsqio/master
# 7. update the release metadata on github / upload the binaries
# 8. update nsqio/nsqio.github.io/_posts/2014-03-01-installing.md
# 9. send release announcement emails
# 10. update IRC channel topic
# 11. tweet
set -e
DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
rm -rf $DIR/dist/docker
mkdir -p $DIR/dist/docker
GOFLAGS='-ldflags="-s -w"'
version=$(awk '/const Binary/ {print $NF}' < $DIR/internal/version/binary.go | sed 's/"//g')
goversion=$(go version | awk '{print $3}')
echo "... running tests"
./test.sh
export GO111MODULE=on
for target in "linux/amd64" "linux/arm64" "darwin/amd64" "darwin/arm64" "freebsd/amd64" "windows/amd64"; do
os=${target%/*}
arch=${target##*/}
echo "... building v$version for $os/$arch"
BUILD=$(mktemp -d ${TMPDIR:-/tmp}/nsq-XXXXX)
TARGET="nsq-$version.$os-$arch.$goversion"
GOOS=$os GOARCH=$arch CGO_ENABLED=0 \
make DESTDIR=$BUILD PREFIX=/$TARGET BLDFLAGS="$GOFLAGS" install
pushd $BUILD
sudo chown -R 0:0 $TARGET
tar czvf $TARGET.tar.gz $TARGET
mv $TARGET.tar.gz $DIR/dist
popd
make clean
sudo rm -r $BUILD
done
rnd=$(LC_ALL=C tr -dc 'a-zA-Z0-9' < /dev/urandom | head -c10)
docker buildx create --use --name nsq-$rnd
docker buildx build .\
--tag nsqio/nsq:v$version \
--platform linux/amd64,linux/arm64 \
--output type=image,push=true
if [[ ! $version == *"-"* ]]; then
echo "Tagging nsqio/nsq:v$version as the latest release"
shas=$(docker manifest inspect nsqio/nsq:v$version |\
grep digest | awk '{print $2}' | sed 's/[",]//g' | sed 's/^/nsqio\/nsq@/')
docker manifest create --amend nsqio/nsq:latest $shas
fi
================================================
FILE: fmt.sh
================================================
#!/bin/bash
find . -name "*.go" | xargs goimports -w
================================================
FILE: go.mod
================================================
module github.com/nsqio/nsq
go 1.17
require (
github.com/BurntSushi/toml v1.3.2
github.com/bitly/go-hostpool v0.1.0
github.com/bitly/timer_metrics v1.0.0
github.com/blang/semver v3.5.1+incompatible
github.com/bmizerany/perks v0.0.0-20141205001514-d9a9656a3a4b
github.com/golang/snappy v0.0.4
github.com/judwhite/go-svc v1.2.1
github.com/julienschmidt/httprouter v1.3.0
github.com/mreiferson/go-options v1.0.0
github.com/nsqio/go-diskqueue v1.1.0
github.com/nsqio/go-nsq v1.1.0
)
require (
github.com/stretchr/testify v1.9.0 // indirect
golang.org/x/sys v0.10.0 // indirect
)
replace github.com/judwhite/go-svc => github.com/mreiferson/go-svc v1.2.2-0.20210815184239-7a96e00010f6
================================================
FILE: go.sum
================================================
github.com/BurntSushi/toml v1.3.2 h1:o7IhLm0Msx3BaB+n3Ag7L8EVlByGnpq14C4YWiu/gL8=
github.com/BurntSushi/toml v1.3.2/go.mod h1:CxXYINrC8qIiEnFrOxCa7Jy5BFHlXnUU2pbicEuybxQ=
github.com/bitly/go-hostpool v0.1.0 h1:XKmsF6k5el6xHG3WPJ8U0Ku/ye7njX7W81Ng7O2ioR0=
github.com/bitly/go-hostpool v0.1.0/go.mod h1:4gOCgp6+NZnVqlKyZ/iBZFTAJKembaVENUpMkpg42fw=
github.com/bitly/timer_metrics v1.0.0 h1:bbszVIl0vT5+/cdZx8L4KOQmM8mC/0y3EBICGSxyhCk=
github.com/bitly/timer_metrics v1.0.0/go.mod h1:87z4/LSg3f++tMqZwZlsLwPuJu6xloyJ7Qm40NyEkLs=
github.com/blang/semver v3.5.1+incompatible h1:cQNTCjp13qL8KC3Nbxr/y2Bqb63oX6wdnnjpJbkM4JQ=
github.com/blang/semver v3.5.1+incompatible/go.mod h1:kRBLl5iJ+tD4TcOOxsy/0fnwebNt5EWlYSAyrTnjyyk=
github.com/bmizerany/perks v0.0.0-20141205001514-d9a9656a3a4b h1:AP/Y7sqYicnjGDfD5VcY4CIfh1hRXBUavxrvELjTiOE=
github.com/bmizerany/perks v0.0.0-20141205001514-d9a9656a3a4b/go.mod h1:ac9efd0D1fsDb3EJvhqgXRbFx7bs2wqZ10HQPeU8U/Q=
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
github.com/golang/snappy v0.0.4 h1:yAGX7huGHXlcLOEtBnF4w7FQwA26wojNCwOYAEhLjQM=
github.com/golang/snappy v0.0.4/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
github.com/julienschmidt/httprouter v1.3.0 h1:U0609e9tgbseu3rBINet9P48AI/D3oJs4dN7jwJOQ1U=
github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8IZAc4RVcycCCAKdM=
github.com/mreiferson/go-options v1.0.0 h1:RMLidydGlDWpL+lQTXo0bVIf/XT2CTq7AEJMoz5/VWs=
github.com/mreiferson/go-options v1.0.0/go.mod h1:zHtCks/HQvOt8ATyfwVe3JJq2PPuImzXINPRTC03+9w=
github.com/mreiferson/go-svc v1.2.2-0.20210815184239-7a96e00010f6 h1:NbuBXARvEXrYZ1SzN53ZpObeuwGhl1zvs/C+kzCggrQ=
github.com/mreiferson/go-svc v1.2.2-0.20210815184239-7a96e00010f6/go.mod h1:mo/P2JNX8C07ywpP9YtO2gnBgnUiFTHqtsZekJrUuTk=
github.com/nsqio/go-diskqueue v1.1.0 h1:r0dJ0DMXT3+2mOq+79cvCjnhoBxyGC2S9O+OjQrpe4Q=
github.com/nsqio/go-diskqueue v1.1.0/go.mod h1:INuJIxl4ayUsyoNtHL5+9MFPDfSZ0zY93hNY6vhBRsI=
github.com/nsqio/go-nsq v1.1.0 h1:PQg+xxiUjA7V+TLdXw7nVrJ5Jbl3sN86EhGCQj4+FYE=
github.com/nsqio/go-nsq v1.1.0/go.mod h1:vKq36oyeVXgsS5Q8YEO7WghqidAVXQlcFxzQbQTuDEY=
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw=
github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo=
github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA=
github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU=
github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo=
github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg=
github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=
golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.10.0 h1:SqMFp9UcQJZa+pmYuAKjd9xq1f0j5rLcDIk0mj4qAsA=
golang.org/x/sys v0.10.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
================================================
FILE: internal/app/float_array.go
================================================
package app
import (
"fmt"
"log"
"sort"
"strconv"
"strings"
)
type FloatArray []float64
func (a *FloatArray) Get() interface{} { return []float64(*a) }
func (a *FloatArray) Set(param string) error {
for _, s := range strings.Split(param, ",") {
v, err := strconv.ParseFloat(s, 64)
if err != nil {
log.Fatalf("Could not parse: %s", s)
return nil
}
*a = append(*a, v)
}
sort.Sort(*a)
return nil
}
func (a FloatArray) Swap(i, j int) { a[i], a[j] = a[j], a[i] }
func (a FloatArray) Less(i, j int) bool { return a[i] > a[j] }
func (a FloatArray) Len() int { return len(a) }
func (a *FloatArray) String() string {
var s []string
for _, v := range *a {
s = append(s, fmt.Sprintf("%f", v))
}
return strings.Join(s, ",")
}
================================================
FILE: internal/app/string_array.go
================================================
package app
import (
"strings"
)
type StringArray []string
func (a *StringArray) Get() interface{} { return []string(*a) }
func (a *StringArray) Set(s string) error {
*a = append(*a, s)
return nil
}
func (a *StringArray) String() string {
return strings.Join(*a, ",")
}
================================================
FILE: internal/auth/authorizations.go
================================================
package auth
import (
"crypto/tls"
"errors"
"fmt"
"math/rand"
"net/url"
"regexp"
"strings"
"time"
"github.com/nsqio/nsq/internal/http_api"
)
type Authorization struct {
Topic string `json:"topic"`
Channels []string `json:"channels"`
Permissions []string `json:"permissions"`
}
type State struct {
TTL int `json:"ttl"`
Authorizations []Authorization `json:"authorizations"`
Identity string `json:"identity"`
IdentityURL string `json:"identity_url"`
Expires time.Time
}
func (a *Authorization) HasPermission(permission string) bool {
for _, p := range a.Permissions {
if permission == p {
return true
}
}
return false
}
func (a *Authorization) IsAllowed(topic, channel string) bool {
if channel != "" {
if !a.HasPermission("subscribe") {
return false
}
} else {
if !a.HasPermission("publish") {
return false
}
}
topicRegex := regexp.MustCompile(a.Topic)
if !topicRegex.MatchString(topic) {
return false
}
for _, c := range a.Channels {
channelRegex := regexp.MustCompile(c)
if channelRegex.MatchString(channel) {
return true
}
}
return false
}
func (a *State) IsAllowed(topic, channel string) bool {
for _, aa := range a.Authorizations {
if aa.IsAllowed(topic, channel) {
return true
}
}
return false
}
func (a *State) IsExpired() bool {
return a.Expires.Before(time.Now())
}
func QueryAnyAuthd(authd []string, remoteIP string, tlsEnabled bool, commonName string, authSecret string,
clientTLSConfig *tls.Config, connectTimeout time.Duration, requestTimeout time.Duration, httpRequestMethod string) (*State, error) {
var retErr error
start := rand.Int()
n := len(authd)
for i := 0; i < n; i++ {
a := authd[(i+start)%n]
authState, err := QueryAuthd(a, remoteIP, tlsEnabled, commonName, authSecret, clientTLSConfig, connectTimeout, requestTimeout, httpRequestMethod)
if err != nil {
es := fmt.Sprintf("failed to auth against %s - %s", a, err)
if retErr != nil {
es = fmt.Sprintf("%s; %s", retErr, es)
}
retErr = errors.New(es)
continue
}
return authState, nil
}
return nil, retErr
}
func QueryAuthd(authd string, remoteIP string, tlsEnabled bool, commonName string, authSecret string,
clientTLSConfig *tls.Config, connectTimeout time.Duration, requestTimeout time.Duration, httpRequestMethod string) (*State, error) {
var authState State
v := url.Values{}
v.Set("remote_ip", remoteIP)
if tlsEnabled {
v.Set("tls", "true")
} else {
v.Set("tls", "false")
}
v.Set("secret", authSecret)
v.Set("common_name", commonName)
var endpoint string
if strings.Contains(authd, "://") {
endpoint = authd
} else {
endpoint = fmt.Sprintf("http://%s/auth", authd)
}
client := http_api.NewClient(clientTLSConfig, connectTimeout, requestTimeout)
if httpRequestMethod == "post" {
if err := client.POSTV1(endpoint, v, &authState); err != nil {
return nil, err
}
} else {
endpoint = fmt.Sprintf("%s?%s", endpoint, v.Encode())
if err := client.GETV1(endpoint, &authState); err != nil {
return nil, err
}
}
// validation on response
for _, auth := range authState.Authorizations {
for _, p := range auth.Permissions {
switch p {
case "subscribe", "publish":
default:
return nil, fmt.Errorf("unknown permission %s", p)
}
}
if _, err := regexp.Compile(auth.Topic); err != nil {
return nil, fmt.Errorf("unable to compile topic %q %s", auth.Topic, err)
}
for _, channel := range auth.Channels {
if _, err := regexp.Compile(channel); err != nil {
return nil, fmt.Errorf("unable to compile channel %q %s", channel, err)
}
}
}
if authState.TTL <= 0 {
return nil, fmt.Errorf("invalid TTL %d (must be >0)", authState.TTL)
}
authState.Expires = time.Now().Add(time.Duration(authState.TTL) * time.Second)
return &authState, nil
}
================================================
FILE: internal/clusterinfo/data.go
================================================
package clusterinfo
import (
"fmt"
"net"
"net/url"
"sort"
"strconv"
"strings"
"sync"
"github.com/blang/semver"
"github.com/nsqio/nsq/internal/http_api"
"github.com/nsqio/nsq/internal/lg"
"github.com/nsqio/nsq/internal/stringy"
)
type PartialErr interface {
error
Errors() []error
}
type ErrList []error
func (l ErrList) Error() string {
var es []string
for _, e := range l {
es = append(es, e.Error())
}
return strings.Join(es, "\n")
}
func (l ErrList) Errors() []error {
return l
}
gitextract_w7hjwstt/ ├── .github/ │ └── workflows/ │ └── test.yml ├── .gitignore ├── AUTHORS ├── CODE_OF_CONDUCT.md ├── CONTRIBUTING.md ├── ChangeLog.md ├── Dockerfile ├── LICENSE ├── Makefile ├── README.md ├── apps/ │ ├── nsq_stat/ │ │ └── nsq_stat.go │ ├── nsq_tail/ │ │ └── nsq_tail.go │ ├── nsq_to_file/ │ │ ├── file_logger.go │ │ ├── nsq_to_file.go │ │ ├── options.go │ │ ├── strftime.go │ │ └── topic_discoverer.go │ ├── nsq_to_http/ │ │ ├── http.go │ │ ├── nsq_to_http.go │ │ └── nsq_to_http_test.go │ ├── nsq_to_nsq/ │ │ └── nsq_to_nsq.go │ ├── nsqadmin/ │ │ ├── main.go │ │ ├── main_test.go │ │ └── options.go │ ├── nsqd/ │ │ ├── README.md │ │ ├── main.go │ │ ├── main_test.go │ │ └── options.go │ ├── nsqlookupd/ │ │ ├── README.md │ │ ├── main.go │ │ ├── main_test.go │ │ └── options.go │ └── to_nsq/ │ ├── README.md │ └── to_nsq.go ├── bench/ │ ├── bench.py │ ├── bench_channels/ │ │ └── bench_channels.go │ ├── bench_reader/ │ │ └── bench_reader.go │ ├── bench_writer/ │ │ └── bench_writer.go │ └── requirements.txt ├── bench.sh ├── contrib/ │ ├── nsq.spec │ ├── nsqadmin.cfg.example │ ├── nsqd.cfg.example │ └── nsqlookupd.cfg.example ├── coverage.sh ├── dist.sh ├── fmt.sh ├── go.mod ├── go.sum ├── internal/ │ ├── app/ │ │ ├── float_array.go │ │ └── string_array.go │ ├── auth/ │ │ └── authorizations.go │ ├── clusterinfo/ │ │ ├── data.go │ │ ├── producer_test.go │ │ └── types.go │ ├── dirlock/ │ │ ├── dirlock.go │ │ ├── dirlock_illumos.go │ │ └── dirlock_windows.go │ ├── http_api/ │ │ ├── api_request.go │ │ ├── api_response.go │ │ ├── compress.go │ │ ├── http_server.go │ │ ├── req_params.go │ │ └── topic_channel_args.go │ ├── lg/ │ │ ├── lg.go │ │ └── lg_test.go │ ├── pqueue/ │ │ ├── pqueue.go │ │ └── pqueue_test.go │ ├── protocol/ │ │ ├── byte_base10.go │ │ ├── byte_base10_test.go │ │ ├── errors.go │ │ ├── names.go │ │ ├── protocol.go │ │ └── tcp_server.go │ ├── quantile/ │ │ ├── aggregate.go │ │ └── quantile.go │ ├── statsd/ │ │ ├── client.go │ │ └── host.go │ ├── stringy/ │ │ ├── slice.go │ │ ├── slice_test.go │ │ └── template.go │ ├── test/ │ │ ├── assertions.go │ │ ├── fakes.go │ │ └── logger.go │ ├── util/ │ │ ├── rand.go │ │ ├── unix_socket.go │ │ ├── util_test.go │ │ └── wait_group_wrapper.go │ ├── version/ │ │ └── binary.go │ └── writers/ │ ├── boundary_buffered_writer.go │ └── spread_writer.go ├── nsqadmin/ │ ├── .eslintrc │ ├── README.md │ ├── gulp │ ├── gulpfile.js │ ├── http.go │ ├── http_test.go │ ├── logger.go │ ├── notify.go │ ├── nsqadmin.go │ ├── nsqadmin_test.go │ ├── options.go │ ├── package.json │ ├── static/ │ │ ├── build/ │ │ │ ├── base.css │ │ │ ├── index.html │ │ │ ├── main.js │ │ │ └── vendor.js │ │ ├── css/ │ │ │ └── base.scss │ │ ├── html/ │ │ │ └── index.html │ │ └── js/ │ │ ├── app_state.js │ │ ├── collections/ │ │ │ ├── nodes.js │ │ │ └── topics.js │ │ ├── lib/ │ │ │ ├── ajax_setup.js │ │ │ ├── handlebars_helpers.js │ │ │ └── pubsub.js │ │ ├── main.js │ │ ├── models/ │ │ │ ├── channel.js │ │ │ ├── node.js │ │ │ └── topic.js │ │ ├── router.js │ │ └── views/ │ │ ├── app.js │ │ ├── base.js │ │ ├── channel.hbs │ │ ├── channel.js │ │ ├── counter.hbs │ │ ├── counter.js │ │ ├── error.hbs │ │ ├── header.hbs │ │ ├── header.js │ │ ├── lookup.hbs │ │ ├── lookup.js │ │ ├── node.hbs │ │ ├── node.js │ │ ├── nodes.hbs │ │ ├── nodes.js │ │ ├── spinner.hbs │ │ ├── topic.hbs │ │ ├── topic.js │ │ ├── topics.hbs │ │ ├── topics.js │ │ └── warning.hbs │ ├── static.go │ └── test/ │ ├── ca.key │ ├── ca.pem │ ├── ca.srl │ ├── cert.pem │ ├── client.key │ ├── client.pem │ ├── client.req │ ├── key.pem │ ├── server.key │ ├── server.pem │ └── server.req ├── nsqd/ │ ├── README.md │ ├── backend_queue.go │ ├── buffer_pool.go │ ├── channel.go │ ├── channel_test.go │ ├── client_v2.go │ ├── dqname.go │ ├── dqname_windows.go │ ├── dummy_backend_queue.go │ ├── guid.go │ ├── guid_test.go │ ├── http.go │ ├── http_test.go │ ├── in_flight_pqueue.go │ ├── in_flight_pqueue_test.go │ ├── logger.go │ ├── lookup.go │ ├── lookup_peer.go │ ├── message.go │ ├── nsqd.go │ ├── nsqd_test.go │ ├── options.go │ ├── protocol_v2.go │ ├── protocol_v2_test.go │ ├── protocol_v2_unixsocket_test.go │ ├── stats.go │ ├── stats_test.go │ ├── statsd.go │ ├── tcp.go │ ├── test/ │ │ ├── cert.sh │ │ ├── certs/ │ │ │ ├── ca.key │ │ │ ├── ca.pem │ │ │ ├── ca.srl │ │ │ ├── cert.pem │ │ │ ├── client.key │ │ │ ├── client.pem │ │ │ ├── client.req │ │ │ ├── key.pem │ │ │ ├── server.key │ │ │ ├── server.pem │ │ │ └── server.req │ │ └── openssl.conf │ ├── topic.go │ └── topic_test.go ├── nsqlookupd/ │ ├── README.md │ ├── client_v1.go │ ├── http.go │ ├── http_test.go │ ├── logger.go │ ├── lookup_protocol_v1.go │ ├── lookup_protocol_v1_test.go │ ├── nsqlookupd.go │ ├── nsqlookupd_test.go │ ├── options.go │ ├── registration_db.go │ ├── registration_db_test.go │ └── tcp.go └── test.sh
SYMBOL INDEX (1336 symbols across 121 files)
FILE: apps/nsq_stat/nsq_stat.go
type numValue (line 36) | type numValue struct
method String (line 41) | func (nv *numValue) String() string { return "N" }
method Set (line 43) | func (nv *numValue) Set(s string) error {
function init (line 53) | func init() {
function statLoop (line 59) | func statLoop(interval time.Duration, connectTimeout time.Duration, requ...
function checkAddrs (line 123) | func checkAddrs(addrs []string) error {
function main (line 132) | func main() {
FILE: apps/nsq_tail/nsq_tail.go
function init (line 31) | func init() {
type TailHandler (line 37) | type TailHandler struct
method HandleMessage (line 43) | func (th *TailHandler) HandleMessage(m *nsq.Message) error {
function main (line 71) | func main() {
FILE: apps/nsq_to_file/file_logger.go
type FileLogger (line 18) | type FileLogger struct
method HandleMessage (line 76) | func (f *FileLogger) HandleMessage(m *nsq.Message) error {
method router (line 82) | func (f *FileLogger) router() {
method Close (line 163) | func (f *FileLogger) Close() {
method Write (line 227) | func (f *FileLogger) Write(p []byte) (int, error) {
method Sync (line 233) | func (f *FileLogger) Sync() error {
method currentFilename (line 251) | func (f *FileLogger) currentFilename() string {
method needsRotation (line 257) | func (f *FileLogger) needsRotation() bool {
method updateFile (line 284) | func (f *FileLogger) updateFile() {
function NewFileLogger (line 40) | func NewFileLogger(logf lg.AppLogFunc, opts *Options, topic string, cfg ...
function makeDirFromPath (line 369) | func makeDirFromPath(logf lg.AppLogFunc, path string) error {
function exclusiveRename (line 377) | func exclusiveRename(src, dst string) error {
function computeFilenameFormat (line 391) | func computeFilenameFormat(opts *Options, topic string) (string, error) {
FILE: apps/nsq_to_file/nsq_to_file.go
function flagSet (line 21) | func flagSet() *flag.FlagSet {
function main (line 61) | func main() {
FILE: apps/nsq_to_file/options.go
type Options (line 5) | type Options struct
function NewOptions (line 33) | func NewOptions() *Options {
FILE: apps/nsq_to_file/strftime.go
function strftime (line 43) | func strftime(format string, t time.Time) string {
FILE: apps/nsq_to_file/topic_discoverer.go
type TopicDiscoverer (line 15) | type TopicDiscoverer struct
method updateTopics (line 39) | func (t *TopicDiscoverer) updateTopics(topics []string) {
method run (line 65) | func (t *TopicDiscoverer) run() {
method isTopicAllowed (line 95) | func (t *TopicDiscoverer) isTopicAllowed(topic string) bool {
function newTopicDiscoverer (line 26) | func newTopicDiscoverer(logf lg.AppLogFunc, opts *Options, cfg *nsq.Conf...
FILE: apps/nsq_to_http/http.go
function init (line 14) | func init() {
function HTTPGet (line 18) | func HTTPGet(endpoint string) (*http.Response, error) {
function HTTPPost (line 30) | func HTTPPost(endpoint string, body *bytes.Buffer) (*http.Response, erro...
FILE: apps/nsq_to_http/nsq_to_http.go
constant ModeAll (line 31) | ModeAll = iota
constant ModeRoundRobin (line 32) | ModeRoundRobin
constant ModeHostPool (line 33) | ModeHostPool
function init (line 59) | func init() {
type Publisher (line 67) | type Publisher interface
type PublishHandler (line 71) | type PublishHandler struct
method HandleMessage (line 84) | func (ph *PublishHandler) HandleMessage(m *nsq.Message) error {
type PostPublisher (line 124) | type PostPublisher struct
method Publish (line 126) | func (p *PostPublisher) Publish(addr string, msg []byte) error {
type GetPublisher (line 141) | type GetPublisher struct
method Publish (line 143) | func (p *GetPublisher) Publish(addr string, msg []byte) error {
function main (line 158) | func main() {
function parseCustomHeaders (line 293) | func parseCustomHeaders(strs []string) (map[string]string, error) {
FILE: apps/nsq_to_http/nsq_to_http_test.go
function TestParseCustomHeaders (line 11) | func TestParseCustomHeaders(t *testing.T) {
FILE: apps/nsq_to_nsq/nsq_to_nsq.go
constant ModeRoundRobin (line 28) | ModeRoundRobin = iota
constant ModeHostPool (line 29) | ModeHostPool
function init (line 51) | func init() {
type PublishHandler (line 59) | type PublishHandler struct
method responder (line 82) | func (ph *PublishHandler) responder() {
method shouldPassMessage (line 123) | func (ph *PublishHandler) shouldPassMessage(js map[string]interface{})...
method HandleMessage (line 205) | func (ph *PublishHandler) HandleMessage(m *nsq.Message, destinationTop...
type TopicHandler (line 77) | type TopicHandler struct
method HandleMessage (line 201) | func (t *TopicHandler) HandleMessage(m *nsq.Message) error {
function filterMessage (line 168) | func filterMessage(js map[string]interface{}, rawMsg []byte) ([]byte, er...
function main (line 257) | func main() {
FILE: apps/nsqadmin/main.go
function nsqadminFlagSet (line 20) | func nsqadminFlagSet(opts *nsqadmin.Options) *flag.FlagSet {
type program (line 66) | type program struct
method Init (line 78) | func (p *program) Init(env svc.Environment) error {
method Start (line 86) | func (p *program) Start() error {
method Stop (line 125) | func (p *program) Stop() error {
function main (line 71) | func main() {
function logFatal (line 132) | func logFatal(f string, args ...interface{}) {
FILE: apps/nsqadmin/main_test.go
function TestConfigFlagParsing (line 12) | func TestConfigFlagParsing(t *testing.T) {
FILE: apps/nsqadmin/options.go
type config (line 9) | type config
method Validate (line 12) | func (cfg config) Validate() {
FILE: apps/nsqd/main.go
type program (line 21) | type program struct
method Init (line 33) | func (p *program) Init(env svc.Environment) error {
method Start (line 68) | func (p *program) Start() error {
method Stop (line 89) | func (p *program) Stop() error {
method Handle (line 96) | func (p *program) Handle(s os.Signal) error {
method Context (line 101) | func (p *program) Context() context.Context {
function main (line 26) | func main() {
function logFatal (line 105) | func logFatal(f string, args ...interface{}) {
function applyBackwardCompatibility (line 110) | func applyBackwardCompatibility(opts *nsqd.Options, flagSet *flag.FlagSe...
FILE: apps/nsqd/main_test.go
function TestConfigFlagParsing (line 15) | func TestConfigFlagParsing(t *testing.T) {
FILE: apps/nsqd/options.go
type tlsRequiredOption (line 15) | type tlsRequiredOption
method Set (line 17) | func (t *tlsRequiredOption) Set(s string) error {
method Get (line 32) | func (t *tlsRequiredOption) Get() interface{} { return int(*t) }
method String (line 34) | func (t *tlsRequiredOption) String() string {
method IsBoolFlag (line 38) | func (t *tlsRequiredOption) IsBoolFlag() bool { return true }
type tlsMinVersionOption (line 40) | type tlsMinVersionOption
method Set (line 52) | func (t *tlsMinVersionOption) Set(s string) error {
method Get (line 66) | func (t *tlsMinVersionOption) Get() interface{} { return uint16(*t) }
method String (line 68) | func (t *tlsMinVersionOption) String() string {
type config (line 77) | type config
method Validate (line 80) | func (cfg config) Validate() {
function nsqdFlagSet (line 116) | func nsqdFlagSet(opts *nsqd.Options) *flag.FlagSet {
FILE: apps/nsqlookupd/main.go
function nsqlookupdFlagSet (line 19) | func nsqlookupdFlagSet(opts *nsqlookupd.Options) *flag.FlagSet {
type program (line 40) | type program struct
method Init (line 52) | func (p *program) Init(env svc.Environment) error {
method Start (line 60) | func (p *program) Start() error {
method Stop (line 99) | func (p *program) Stop() error {
function main (line 45) | func main() {
function logFatal (line 106) | func logFatal(f string, args ...interface{}) {
FILE: apps/nsqlookupd/main_test.go
function TestConfigFlagParsing (line 12) | func TestConfigFlagParsing(t *testing.T) {
FILE: apps/nsqlookupd/options.go
type config (line 9) | type config
method Validate (line 12) | func (cfg config) Validate() {
FILE: apps/to_nsq/to_nsq.go
function init (line 30) | func init() {
function main (line 34) | func main() {
function readAndPublish (line 128) | func readAndPublish(r *bufio.Reader, delim byte, producers map[string]*n...
FILE: bench/bench.py
function ssh_connect_with_retries (line 34) | def ssh_connect_with_retries(host, retries=3, timeout=30):
function ssh_cmd_async (line 48) | def ssh_cmd_async(ssh_client, cmd):
function ssh_cmd (line 55) | def ssh_cmd(ssh_client, cmd, timeout=2):
function get_session (line 81) | def get_session():
function _bootstrap (line 85) | def _bootstrap(addr):
function bootstrap (line 105) | def bootstrap():
function run (line 144) | def run():
function _find_instances (line 263) | def _find_instances():
function decomm (line 270) | def decomm():
FILE: bench/bench_channels/bench_channels.go
function main (line 19) | func main() {
function subWorker (line 39) | func subWorker(n int, tcpAddr string,
FILE: bench/bench_reader/bench_reader.go
function main (line 30) | func main() {
function subWorker (line 71) | func subWorker(td time.Duration, workers int, tcpAddr string, topic stri...
FILE: bench/bench_writer/bench_writer.go
function main (line 28) | func main() {
function pubWorker (line 74) | func pubWorker(td time.Duration, tcpAddr string, batchSize int, batch []...
FILE: internal/app/float_array.go
type FloatArray (line 11) | type FloatArray
method Get (line 13) | func (a *FloatArray) Get() interface{} { return []float64(*a) }
method Set (line 15) | func (a *FloatArray) Set(param string) error {
method Swap (line 28) | func (a FloatArray) Swap(i, j int) { a[i], a[j] = a[j], a[i] }
method Less (line 29) | func (a FloatArray) Less(i, j int) bool { return a[i] > a[j] }
method Len (line 30) | func (a FloatArray) Len() int { return len(a) }
method String (line 32) | func (a *FloatArray) String() string {
FILE: internal/app/string_array.go
type StringArray (line 7) | type StringArray
method Get (line 9) | func (a *StringArray) Get() interface{} { return []string(*a) }
method Set (line 11) | func (a *StringArray) Set(s string) error {
method String (line 16) | func (a *StringArray) String() string {
FILE: internal/auth/authorizations.go
type Authorization (line 16) | type Authorization struct
method HasPermission (line 30) | func (a *Authorization) HasPermission(permission string) bool {
method IsAllowed (line 39) | func (a *Authorization) IsAllowed(topic, channel string) bool {
type State (line 22) | type State struct
method IsAllowed (line 65) | func (a *State) IsAllowed(topic, channel string) bool {
method IsExpired (line 74) | func (a *State) IsExpired() bool {
function QueryAnyAuthd (line 78) | func QueryAnyAuthd(authd []string, remoteIP string, tlsEnabled bool, com...
function QueryAuthd (line 99) | func QueryAuthd(authd string, remoteIP string, tlsEnabled bool, commonNa...
FILE: internal/clusterinfo/data.go
type PartialErr (line 18) | type PartialErr interface
type ErrList (line 23) | type ErrList
method Error (line 25) | func (l ErrList) Error() string {
method Errors (line 33) | func (l ErrList) Errors() []error {
type ClusterInfo (line 37) | type ClusterInfo struct
method logf (line 49) | func (c *ClusterInfo) logf(f string, args ...interface{}) {
method GetVersion (line 56) | func (c *ClusterInfo) GetVersion(addr string) (semver.Version, error) {
method GetLookupdTopics (line 73) | func (c *ClusterInfo) GetLookupdTopics(lookupdHTTPAddrs []string) ([]s...
method GetLookupdTopicChannels (line 122) | func (c *ClusterInfo) GetLookupdTopicChannels(topic string, lookupdHTT...
method GetLookupdProducers (line 170) | func (c *ClusterInfo) GetLookupdProducers(lookupdHTTPAddrs []string) (...
method GetLookupdTopicProducers (line 240) | func (c *ClusterInfo) GetLookupdTopicProducers(topic string, lookupdHT...
method GetNSQDTopics (line 292) | func (c *ClusterInfo) GetNSQDTopics(nsqdHTTPAddrs []string) ([]string,...
method GetNSQDProducers (line 343) | func (c *ClusterInfo) GetNSQDProducers(nsqdHTTPAddrs []string) (Produc...
method GetNSQDTopicProducers (line 432) | func (c *ClusterInfo) GetNSQDTopicProducers(topic string, nsqdHTTPAddr...
method GetNSQDStats (line 544) | func (c *ClusterInfo) GetNSQDStats(producers Producers,
method TombstoneNodeForTopic (line 644) | func (c *ClusterInfo) TombstoneNodeForTopic(topic string, node string,...
method CreateTopicChannel (line 684) | func (c *ClusterInfo) CreateTopicChannel(topicName string, channelName...
method DeleteTopic (line 736) | func (c *ClusterInfo) DeleteTopic(topicName string, lookupdHTTPAddrs [...
method DeleteChannel (line 777) | func (c *ClusterInfo) DeleteChannel(topicName string, channelName stri...
method PauseTopic (line 817) | func (c *ClusterInfo) PauseTopic(topicName string, lookupdHTTPAddrs []...
method UnPauseTopic (line 822) | func (c *ClusterInfo) UnPauseTopic(topicName string, lookupdHTTPAddrs ...
method PauseChannel (line 827) | func (c *ClusterInfo) PauseChannel(topicName string, channelName strin...
method UnPauseChannel (line 832) | func (c *ClusterInfo) UnPauseChannel(topicName string, channelName str...
method EmptyTopic (line 837) | func (c *ClusterInfo) EmptyTopic(topicName string, lookupdHTTPAddrs []...
method EmptyChannel (line 842) | func (c *ClusterInfo) EmptyChannel(topicName string, channelName strin...
method actionHelper (line 847) | func (c *ClusterInfo) actionHelper(topicName string, lookupdHTTPAddrs ...
method GetProducers (line 874) | func (c *ClusterInfo) GetProducers(lookupdHTTPAddrs []string, nsqdHTTP...
method GetTopicProducers (line 881) | func (c *ClusterInfo) GetTopicProducers(topicName string, lookupdHTTPA...
method nsqlookupdPOST (line 888) | func (c *ClusterInfo) nsqlookupdPOST(addrs []string, uri string, qs st...
method producersPOST (line 904) | func (c *ClusterInfo) producersPOST(pl Producers, uri string, qs strin...
function New (line 42) | func New(log lg.AppLogFunc, client *http_api.Client) *ClusterInfo {
FILE: internal/clusterinfo/producer_test.go
function TestHostNameAddresses (line 5) | func TestHostNameAddresses(t *testing.T) {
function TestIPv4Addresses (line 20) | func TestIPv4Addresses(t *testing.T) {
function TestIPv6Addresses (line 35) | func TestIPv6Addresses(t *testing.T) {
FILE: internal/clusterinfo/types.go
type ProducerTopic (line 14) | type ProducerTopic struct
type ProducerTopics (line 19) | type ProducerTopics
method Len (line 21) | func (pt ProducerTopics) Len() int { return len(pt) }
method Swap (line 22) | func (pt ProducerTopics) Swap(i, j int) { pt[i], pt[j] = pt[j], p...
method Less (line 23) | func (pt ProducerTopics) Less(i, j int) bool { return pt[i].Topic < pt...
type Producer (line 25) | type Producer struct
method UnmarshalJSON (line 41) | func (p *Producer) UnmarshalJSON(b []byte) error {
method Address (line 78) | func (p *Producer) Address() string {
method HTTPAddress (line 85) | func (p *Producer) HTTPAddress() string {
method TCPAddress (line 89) | func (p *Producer) TCPAddress() string {
method IsInconsistent (line 96) | func (p *Producer) IsInconsistent(numLookupd int) bool {
type TopicStats (line 100) | type TopicStats struct
method Add (line 119) | func (t *TopicStats) Add(a *TopicStats) {
type ChannelStats (line 155) | type ChannelStats struct
method Add (line 181) | func (c *ChannelStats) Add(a *ChannelStats) {
type ClientStats (line 213) | type ClientStats struct
method UnmarshalJSON (line 246) | func (s *ClientStats) UnmarshalJSON(b []byte) error {
method HasUserAgent (line 257) | func (s *ClientStats) HasUserAgent() bool {
method HasSampleRate (line 261) | func (s *ClientStats) HasSampleRate() bool {
type ChannelStatsList (line 265) | type ChannelStatsList
method Len (line 267) | func (c ChannelStatsList) Len() int { return len(c) }
method Swap (line 268) | func (c ChannelStatsList) Swap(i, j int) { c[i], c[j] = c[j], c[i] }
type ChannelStatsByHost (line 270) | type ChannelStatsByHost struct
method Less (line 274) | func (c ChannelStatsByHost) Less(i, j int) bool {
type ClientStatsList (line 278) | type ClientStatsList
method Len (line 280) | func (c ClientStatsList) Len() int { return len(c) }
method Swap (line 281) | func (c ClientStatsList) Swap(i, j int) { c[i], c[j] = c[j], c[i] }
type ClientsByHost (line 283) | type ClientsByHost struct
method Less (line 287) | func (c ClientsByHost) Less(i, j int) bool {
type ClientStatsByNodeTopology (line 291) | type ClientStatsByNodeTopology struct
method Less (line 295) | func (c ClientStatsByNodeTopology) Less(i, j int) bool {
type TopicStatsList (line 320) | type TopicStatsList
method Len (line 322) | func (t TopicStatsList) Len() int { return len(t) }
method Swap (line 323) | func (t TopicStatsList) Swap(i, j int) { t[i], t[j] = t[j], t[i] }
type TopicStatsByHost (line 325) | type TopicStatsByHost struct
method Less (line 329) | func (c TopicStatsByHost) Less(i, j int) bool {
type Producers (line 333) | type Producers
method Len (line 335) | func (t Producers) Len() int { return len(t) }
method Swap (line 336) | func (t Producers) Swap(i, j int) { t[i], t[j] = t[j], t[i] }
method HTTPAddrs (line 338) | func (t Producers) HTTPAddrs() []string {
method Search (line 346) | func (t Producers) Search(needle string) *Producer {
type ProducersByHost (line 355) | type ProducersByHost struct
method Less (line 359) | func (c ProducersByHost) Less(i, j int) bool {
FILE: internal/dirlock/dirlock.go
type DirLock (line 12) | type DirLock struct
method Lock (line 23) | func (l *DirLock) Lock() error {
method Unlock (line 36) | func (l *DirLock) Unlock() error {
function New (line 17) | func New(dir string) *DirLock {
FILE: internal/dirlock/dirlock_illumos.go
type DirLock (line 6) | type DirLock struct
method Lock (line 16) | func (l *DirLock) Lock() error {
method Unlock (line 20) | func (l *DirLock) Unlock() error {
function New (line 10) | func New(dir string) *DirLock {
FILE: internal/dirlock/dirlock_windows.go
type DirLock (line 6) | type DirLock struct
method Lock (line 16) | func (l *DirLock) Lock() error {
method Unlock (line 20) | func (l *DirLock) Unlock() error {
function New (line 10) | func New(dir string) *DirLock {
FILE: internal/http_api/api_request.go
function NewDeadlineTransport (line 18) | func NewDeadlineTransport(connectTimeout time.Duration, requestTimeout t...
type Client (line 34) | type Client struct
method GETV1 (line 51) | func (c *Client) GETV1(endpoint string, v interface{}) error {
method POSTV1 (line 90) | func (c *Client) POSTV1(endpoint string, data url.Values, v interface{...
function NewClient (line 38) | func NewClient(tlsConfig *tls.Config, connectTimeout time.Duration, requ...
function httpsEndpoint (line 139) | func httpsEndpoint(endpoint string, body []byte) (string, error) {
FILE: internal/http_api/api_response.go
type Decorator (line 14) | type Decorator
type APIHandler (line 16) | type APIHandler
type Err (line 18) | type Err struct
method Error (line 23) | func (e Err) Error() string {
function PlainText (line 27) | func PlainText(f APIHandler) APIHandler {
function V1 (line 49) | func V1(f APIHandler) APIHandler {
function RespondV1 (line 61) | func RespondV1(w http.ResponseWriter, code int, data interface{}) {
function Decorate (line 99) | func Decorate(f APIHandler, ds ...Decorator) httprouter.Handle {
function Log (line 109) | func Log(logf lg.AppLogFunc) Decorator {
function LogPanicHandler (line 126) | func LogPanicHandler(logf lg.AppLogFunc) func(w http.ResponseWriter, req...
function LogNotFoundHandler (line 135) | func LogNotFoundHandler(logf lg.AppLogFunc) http.Handler {
function LogMethodNotAllowedHandler (line 143) | func LogMethodNotAllowedHandler(logf lg.AppLogFunc) http.Handler {
FILE: internal/http_api/compress.go
type compressResponseWriter (line 17) | type compressResponseWriter struct
method Header (line 23) | func (w *compressResponseWriter) Header() http.Header {
method WriteHeader (line 27) | func (w *compressResponseWriter) WriteHeader(c int) {
method Write (line 32) | func (w *compressResponseWriter) Write(b []byte) (int, error) {
function CompressHandler (line 43) | func CompressHandler(h http.Handler) http.Handler {
FILE: internal/http_api/http_server.go
type logWriter (line 13) | type logWriter struct
method Write (line 17) | func (l logWriter) Write(p []byte) (int, error) {
function Serve (line 22) | func Serve(listener net.Listener, handler http.Handler, proto string, lo...
FILE: internal/http_api/req_params.go
type ReqParams (line 10) | type ReqParams struct
method Get (line 29) | func (r *ReqParams) Get(key string) (string, error) {
method GetAll (line 37) | func (r *ReqParams) GetAll(key string) ([]string, error) {
function NewReqParams (line 15) | func NewReqParams(req *http.Request) (*ReqParams, error) {
FILE: internal/http_api/topic_channel_args.go
type getter (line 9) | type getter interface
function GetTopicChannelArgs (line 13) | func GetTopicChannelArgs(rp getter) (string, string, error) {
FILE: internal/lg/lg.go
constant DEBUG (line 12) | DEBUG = LogLevel(1)
constant INFO (line 13) | INFO = LogLevel(2)
constant WARN (line 14) | WARN = LogLevel(3)
constant ERROR (line 15) | ERROR = LogLevel(4)
constant FATAL (line 16) | FATAL = LogLevel(5)
type AppLogFunc (line 19) | type AppLogFunc
type Logger (line 21) | type Logger interface
type NilLogger (line 25) | type NilLogger struct
method Output (line 27) | func (l NilLogger) Output(maxdepth int, s string) error {
type LogLevel (line 31) | type LogLevel
method Get (line 33) | func (l *LogLevel) Get() interface{} { return *l }
method Set (line 35) | func (l *LogLevel) Set(s string) error {
method String (line 44) | func (l *LogLevel) String() string {
function ParseLogLevel (line 60) | func ParseLogLevel(levelstr string) (LogLevel, error) {
function Logf (line 76) | func Logf(logger Logger, cfgLevel LogLevel, msgLevel LogLevel, f string,...
function LogFatal (line 83) | func LogFatal(prefix string, f string, args ...interface{}) {
FILE: internal/lg/lg_test.go
type mockLogger (line 9) | type mockLogger struct
method Output (line 13) | func (l *mockLogger) Output(maxdepth int, s string) error {
function TestLogging (line 18) | func TestLogging(t *testing.T) {
FILE: internal/pqueue/pqueue.go
type Item (line 7) | type Item struct
type PriorityQueue (line 15) | type PriorityQueue
method Len (line 21) | func (pq PriorityQueue) Len() int {
method Less (line 25) | func (pq PriorityQueue) Less(i, j int) bool {
method Swap (line 29) | func (pq PriorityQueue) Swap(i, j int) {
method Push (line 35) | func (pq *PriorityQueue) Push(x interface{}) {
method Pop (line 49) | func (pq *PriorityQueue) Pop() interface{} {
method PeekAndShift (line 63) | func (pq *PriorityQueue) PeekAndShift(max int64) (*Item, int64) {
function New (line 17) | func New(capacity int) PriorityQueue {
FILE: internal/pqueue/pqueue_test.go
function equal (line 13) | func equal(t *testing.T, act, exp interface{}) {
function TestPriorityQueue (line 22) | func TestPriorityQueue(t *testing.T) {
function TestUnsortedInsert (line 39) | func TestUnsortedInsert(t *testing.T) {
function TestRemove (line 60) | func TestRemove(t *testing.T) {
FILE: internal/protocol/byte_base10.go
function ByteToBase10 (line 9) | func ByteToBase10(b []byte) (n uint64, err error) {
FILE: internal/protocol/byte_base10_test.go
function BenchmarkByteToBase10Valid (line 9) | func BenchmarkByteToBase10Valid(b *testing.B) {
function BenchmarkByteToBase10Invalid (line 18) | func BenchmarkByteToBase10Invalid(b *testing.B) {
FILE: internal/protocol/errors.go
type ChildErr (line 3) | type ChildErr interface
type ClientErr (line 11) | type ClientErr struct
method Error (line 18) | func (e *ClientErr) Error() string {
method Parent (line 23) | func (e *ClientErr) Parent() error {
function NewClientErr (line 28) | func NewClientErr(parent error, code string, description string) *Client...
type FatalClientErr (line 32) | type FatalClientErr struct
method Error (line 39) | func (e *FatalClientErr) Error() string {
method Parent (line 44) | func (e *FatalClientErr) Parent() error {
function NewFatalClientErr (line 49) | func NewFatalClientErr(parent error, code string, description string) *F...
FILE: internal/protocol/names.go
function IsValidTopicName (line 10) | func IsValidTopicName(name string) bool {
function IsValidChannelName (line 15) | func IsValidChannelName(name string) bool {
function isValidName (line 19) | func isValidName(name string) bool {
FILE: internal/protocol/protocol.go
type Client (line 9) | type Client interface
type Protocol (line 14) | type Protocol interface
function SendResponse (line 21) | func SendResponse(w io.Writer, data []byte) (int, error) {
function SendFramedResponse (line 37) | func SendFramedResponse(w io.Writer, frameType int32, data []byte) (int,...
FILE: internal/protocol/tcp_server.go
type TCPHandler (line 13) | type TCPHandler interface
function TCPServer (line 17) | func TCPServer(listener net.Listener, handler TCPHandler, logf lg.AppLog...
FILE: internal/quantile/aggregate.go
type E2eProcessingLatencyAggregate (line 9) | type E2eProcessingLatencyAggregate struct
method UnmarshalJSON (line 17) | func (e *E2eProcessingLatencyAggregate) UnmarshalJSON(b []byte) error {
method Len (line 46) | func (e *E2eProcessingLatencyAggregate) Len() int { return len(e.Perce...
method Swap (line 47) | func (e *E2eProcessingLatencyAggregate) Swap(i, j int) {
method Less (line 50) | func (e *E2eProcessingLatencyAggregate) Less(i, j int) bool {
method Add (line 55) | func (e *E2eProcessingLatencyAggregate) Add(e2 *E2eProcessingLatencyAg...
FILE: internal/quantile/quantile.go
type Result (line 12) | type Result struct
method String (line 17) | func (r *Result) String() string {
type Quantile (line 25) | type Quantile struct
method Result (line 50) | func (q *Quantile) Result() *Result {
method Insert (line 66) | func (q *Quantile) Insert(msgStartTime int64) {
method QueryHandler (line 78) | func (q *Quantile) QueryHandler() *quantile.Stream {
method IsDataStale (line 92) | func (q *Quantile) IsDataStale(now time.Time) bool {
method Merge (line 96) | func (q *Quantile) Merge(them *Quantile) {
method moveWindow (line 115) | func (q *Quantile) moveWindow() {
function New (line 36) | func New(WindowTime time.Duration, Percentiles []float64) *Quantile {
FILE: internal/statsd/client.go
type Client (line 8) | type Client struct
method Incr (line 20) | func (c *Client) Incr(stat string, count int64) error {
method Decr (line 24) | func (c *Client) Decr(stat string, count int64) error {
method Timing (line 28) | func (c *Client) Timing(stat string, delta int64) error {
method Gauge (line 32) | func (c *Client) Gauge(stat string, value int64) error {
method send (line 36) | func (c *Client) send(stat string, format string, value int64) error {
function NewClient (line 13) | func NewClient(w io.Writer, prefix string) *Client {
FILE: internal/statsd/host.go
function HostKey (line 7) | func HostKey(h string) string {
FILE: internal/stringy/slice.go
function Add (line 3) | func Add(s []string, a string) []string {
function Union (line 13) | func Union(s []string, a []string) []string {
function Uniq (line 29) | func Uniq(s []string) (r []string) {
FILE: internal/stringy/slice_test.go
function BenchmarkUniq (line 9) | func BenchmarkUniq(b *testing.B) {
function TestUniq (line 19) | func TestUniq(t *testing.T) {
FILE: internal/stringy/template.go
function NanoSecondToHuman (line 7) | func NanoSecondToHuman(v float64) string {
FILE: internal/test/assertions.go
function Equal (line 10) | func Equal(t *testing.T, expected, actual interface{}) {
function NotEqual (line 19) | func NotEqual(t *testing.T, expected, actual interface{}) {
function Nil (line 28) | func Nil(t *testing.T, object interface{}) {
function NotNil (line 37) | func NotNil(t *testing.T, object interface{}) {
function isNil (line 46) | func isNil(object interface{}) bool {
FILE: internal/test/fakes.go
type FakeNetConn (line 8) | type FakeNetConn struct
method Read (line 19) | func (f FakeNetConn) Read(b []byte) (int, error) { return f.Re...
method Write (line 20) | func (f FakeNetConn) Write(b []byte) (int, error) { return f.Wr...
method Close (line 21) | func (f FakeNetConn) Close() error { return f.Cl...
method LocalAddr (line 22) | func (f FakeNetConn) LocalAddr() net.Addr { return f.Lo...
method RemoteAddr (line 23) | func (f FakeNetConn) RemoteAddr() net.Addr { return f.Re...
method SetDeadline (line 24) | func (f FakeNetConn) SetDeadline(t time.Time) error { return f.Se...
method SetReadDeadline (line 25) | func (f FakeNetConn) SetReadDeadline(t time.Time) error { return f.Se...
method SetWriteDeadline (line 26) | func (f FakeNetConn) SetWriteDeadline(t time.Time) error { return f.Se...
type fakeNetAddr (line 28) | type fakeNetAddr struct
method Network (line 30) | func (fakeNetAddr) Network() string { return "" }
method String (line 31) | func (fakeNetAddr) String() string { return "" }
function NewFakeNetConn (line 33) | func NewFakeNetConn() FakeNetConn {
FILE: internal/test/logger.go
type Logger (line 3) | type Logger interface
type tbLog (line 7) | type tbLog interface
type testLogger (line 11) | type testLogger struct
method Output (line 15) | func (tl *testLogger) Output(maxdepth int, s string) error {
function NewTestLogger (line 20) | func NewTestLogger(tbl tbLog) Logger {
FILE: internal/util/rand.go
function UniqRands (line 7) | func UniqRands(quantity int, maxval int) []int {
FILE: internal/util/unix_socket.go
function TypeOfAddr (line 7) | func TypeOfAddr(addr string) string {
FILE: internal/util/util_test.go
function BenchmarkUniqRands5of5 (line 9) | func BenchmarkUniqRands5of5(b *testing.B) {
function BenchmarkUniqRands20of20 (line 14) | func BenchmarkUniqRands20of20(b *testing.B) {
function BenchmarkUniqRands20of50 (line 20) | func BenchmarkUniqRands20of50(b *testing.B) {
function TestUniqRands (line 26) | func TestUniqRands(t *testing.T) {
function TestTypeOfAddr (line 38) | func TestTypeOfAddr(t *testing.T) {
FILE: internal/util/wait_group_wrapper.go
type WaitGroupWrapper (line 7) | type WaitGroupWrapper struct
method Wrap (line 11) | func (w *WaitGroupWrapper) Wrap(cb func()) {
FILE: internal/version/binary.go
constant Binary (line 8) | Binary = "1.3.0"
function String (line 10) | func String(app string) string {
FILE: internal/writers/boundary_buffered_writer.go
type BoundaryBufferedWriter (line 8) | type BoundaryBufferedWriter struct
method Write (line 18) | func (b *BoundaryBufferedWriter) Write(p []byte) (int, error) {
method Flush (line 28) | func (b *BoundaryBufferedWriter) Flush() error {
function NewBoundaryBufferedWriter (line 12) | func NewBoundaryBufferedWriter(w io.Writer, size int) *BoundaryBufferedW...
FILE: internal/writers/spread_writer.go
type SpreadWriter (line 8) | type SpreadWriter struct
method Write (line 24) | func (s *SpreadWriter) Write(p []byte) (int, error) {
method Flush (line 31) | func (s *SpreadWriter) Flush() {
function NewSpreadWriter (line 15) | func NewSpreadWriter(w io.Writer, interval time.Duration, exitCh chan in...
FILE: nsqadmin/gulpfile.js
function excludeVendor (line 27) | function excludeVendor(b) {
function bytesToKB (line 33) | function bytesToKB(bytes) { return Math.floor(+bytes/1024); }
function logBundle (line 35) | function logBundle(filename, watching) {
function sassTask (line 50) | function sassTask(root, inputFile) {
function browserifyTask (line 64) | function browserifyTask(root, inputFile) {
function watchTask (line 98) | function watchTask(root) {
function cleanTask (line 113) | function cleanTask() {
FILE: nsqadmin/http.go
function maybeWarnMsg (line 28) | func maybeWarnMsg(msgs []string) string {
function NewSingleHostReverseProxy (line 36) | func NewSingleHostReverseProxy(target *url.URL, connectTimeout time.Dura...
type httpServer (line 51) | type httpServer struct
method ServeHTTP (line 125) | func (s *httpServer) ServeHTTP(w http.ResponseWriter, req *http.Reques...
method pingHandler (line 129) | func (s *httpServer) pingHandler(w http.ResponseWriter, req *http.Requ...
method indexHandler (line 133) | func (s *httpServer) indexHandler(w http.ResponseWriter, req *http.Req...
method staticAssetHandler (line 169) | func (s *httpServer) staticAssetHandler(w http.ResponseWriter, req *ht...
method topicsHandler (line 212) | func (s *httpServer) topicsHandler(w http.ResponseWriter, req *http.Re...
method topicHandler (line 264) | func (s *httpServer) topicHandler(w http.ResponseWriter, req *http.Req...
method channelHandler (line 303) | func (s *httpServer) channelHandler(w http.ResponseWriter, req *http.R...
method nodesHandler (line 340) | func (s *httpServer) nodesHandler(w http.ResponseWriter, req *http.Req...
method nodeHandler (line 360) | func (s *httpServer) nodeHandler(w http.ResponseWriter, req *http.Requ...
method tombstoneNodeForTopicHandler (line 411) | func (s *httpServer) tombstoneNodeForTopicHandler(w http.ResponseWrite...
method createTopicChannelHandler (line 451) | func (s *httpServer) createTopicChannelHandler(w http.ResponseWriter, ...
method deleteTopicHandler (line 498) | func (s *httpServer) deleteTopicHandler(w http.ResponseWriter, req *ht...
method deleteChannelHandler (line 527) | func (s *httpServer) deleteChannelHandler(w http.ResponseWriter, req *...
method topicActionHandler (line 557) | func (s *httpServer) topicActionHandler(w http.ResponseWriter, req *ht...
method channelActionHandler (line 562) | func (s *httpServer) channelActionHandler(w http.ResponseWriter, req *...
method topicChannelAction (line 568) | func (s *httpServer) topicChannelAction(req *http.Request, topicName s...
method counterHandler (line 653) | func (s *httpServer) counterHandler(w http.ResponseWriter, req *http.R...
method graphiteHandler (line 700) | func (s *httpServer) graphiteHandler(w http.ResponseWriter, req *http....
method doConfig (line 749) | func (s *httpServer) doConfig(w http.ResponseWriter, req *http.Request...
method isAuthorizedAdminRequest (line 810) | func (s *httpServer) isAuthorizedAdminRequest(req *http.Request) bool {
function NewHTTPServer (line 60) | func NewHTTPServer(nsqadmin *NSQAdmin) *httpServer {
type counterStats (line 646) | type counterStats struct
function getOptByCfgName (line 825) | func getOptByCfgName(opts interface{}, name string) (interface{}, bool) {
FILE: nsqadmin/http_test.go
type TopicsDoc (line 22) | type TopicsDoc struct
type TopicStatsDoc (line 26) | type TopicStatsDoc struct
type NodesDoc (line 31) | type NodesDoc struct
type NodeStatsDoc (line 36) | type NodeStatsDoc struct
type ChannelStatsDoc (line 44) | type ChannelStatsDoc struct
function mustStartNSQLookupd (line 49) | func mustStartNSQLookupd(opts *nsqlookupd.Options) (*net.TCPAddr, *net.T...
function bootstrapNSQCluster (line 65) | func bootstrapNSQCluster(t *testing.T) (string, []*nsqd.NSQD, []*nsqlook...
function bootstrapNSQClusterWithAuth (line 69) | func bootstrapNSQClusterWithAuth(t *testing.T, withAuth bool) (string, [...
function TestPing (line 135) | func TestPing(t *testing.T) {
function TestHTTPTopicsGET (line 154) | func TestHTTPTopicsGET(t *testing.T) {
function TestHTTPTopicGET (line 182) | func TestHTTPTopicGET(t *testing.T) {
function TestHTTPNodesGET (line 214) | func TestHTTPNodesGET(t *testing.T) {
function TestHTTPChannelGET (line 248) | func TestHTTPChannelGET(t *testing.T) {
function TestHTTPNodesSingleGET (line 287) | func TestHTTPNodesSingleGET(t *testing.T) {
function TestHTTPCreateTopicPOST (line 324) | func TestHTTPCreateTopicPOST(t *testing.T) {
function TestHTTPCreateTopicChannelPOST (line 347) | func TestHTTPCreateTopicChannelPOST(t *testing.T) {
function TestHTTPTombstoneTopicNodePOST (line 371) | func TestHTTPTombstoneTopicNodePOST(t *testing.T) {
function TestHTTPDeleteTopicPOST (line 394) | func TestHTTPDeleteTopicPOST(t *testing.T) {
function TestHTTPDeleteChannelPOST (line 414) | func TestHTTPDeleteChannelPOST(t *testing.T) {
function TestHTTPPauseTopicPOST (line 435) | func TestHTTPPauseTopicPOST(t *testing.T) {
function TestHTTPPauseChannelPOST (line 469) | func TestHTTPPauseChannelPOST(t *testing.T) {
function TestHTTPEmptyTopicPOST (line 504) | func TestHTTPEmptyTopicPOST(t *testing.T) {
function TestHTTPEmptyChannelPOST (line 532) | func TestHTTPEmptyChannelPOST(t *testing.T) {
function TestHTTPconfig (line 562) | func TestHTTPconfig(t *testing.T) {
function TestHTTPconfigCIDR (line 620) | func TestHTTPconfigCIDR(t *testing.T) {
FILE: nsqadmin/logger.go
type Logger (line 7) | type Logger
constant LOG_DEBUG (line 10) | LOG_DEBUG = lg.DEBUG
constant LOG_INFO (line 11) | LOG_INFO = lg.INFO
constant LOG_WARN (line 12) | LOG_WARN = lg.WARN
constant LOG_ERROR (line 13) | LOG_ERROR = lg.ERROR
constant LOG_FATAL (line 14) | LOG_FATAL = lg.FATAL
method logf (line 17) | func (n *NSQAdmin) logf(level lg.LogLevel, f string, args ...interface{}) {
FILE: nsqadmin/notify.go
type AdminAction (line 12) | type AdminAction struct
function basicAuthUser (line 25) | func basicAuthUser(req *http.Request) string {
method notifyAdminAction (line 41) | func (s *httpServer) notifyAdminAction(action, topic, channel, node stri...
FILE: nsqadmin/nsqadmin.go
type NSQAdmin (line 24) | type NSQAdmin struct
method getOpts (line 137) | func (n *NSQAdmin) getOpts() *Options {
method swapOpts (line 141) | func (n *NSQAdmin) swapOpts(opts *Options) {
method RealHTTPAddr (line 145) | func (n *NSQAdmin) RealHTTPAddr() *net.TCPAddr {
method handleAdminActions (line 149) | func (n *NSQAdmin) handleAdminActions() {
method Main (line 168) | func (n *NSQAdmin) Main() error {
method Exit (line 190) | func (n *NSQAdmin) Exit() {
function New (line 34) | func New(opts *Options) (*NSQAdmin, error) {
function normalizeBasePath (line 126) | func normalizeBasePath(p string) string {
FILE: nsqadmin/nsqadmin_test.go
function TestNeitherNSQDAndNSQLookup (line 16) | func TestNeitherNSQDAndNSQLookup(t *testing.T) {
function TestBothNSQDAndNSQLookup (line 25) | func TestBothNSQDAndNSQLookup(t *testing.T) {
function TestTLSHTTPClient (line 36) | func TestTLSHTTPClient(t *testing.T) {
function mustStartNSQD (line 80) | func mustStartNSQD(opts *nsqd.Options) (net.Addr, net.Addr, *nsqd.NSQD) {
FILE: nsqadmin/options.go
type Options (line 9) | type Options struct
function NewOptions (line 47) | func NewOptions() *Options {
FILE: nsqadmin/static.go
function staticAsset (line 10) | func staticAsset(name string) ([]byte, error) {
FILE: nsqadmin/static/build/main.js
function i (line 1) | function i(e,n){if(!o[e]){if(!a[e]){var t="function"==typeof require&&re...
function t (line 1) | function t(t){return this.each(function(){var n=r(this),e=n.data("bs.aff...
function r (line 1) | function r(n){o(n).on("click",e,this.close)}
function a (line 1) | function a(){l.detach().trigger("closed.bs.alert").remove()}
function t (line 1) | function t(t){return this.each(function(){var n=o(this),e=n.data("bs.but...
function u (line 1) | function u(n,e){this.$element=c(n),this.$indicators=this.$element.find("...
function a (line 1) | function a(a){return this.each(function(){var n=c(this),e=n.data("bs.car...
function e (line 1) | function e(n){var e=c(this),t=(t=e.attr("href"))&&t.replace(/.*(?=#[^\s]...
function t (line 1) | function t(n){var e=n.attr("data-target")||(e=n.attr("href"))&&e.replace...
function r (line 1) | function r(l){return this.each(function(){var n=a(this),e=n.data("bs.col...
function l (line 1) | function l(n){a(n).on("click.bs.dropdown",this.toggle)}
function r (line 1) | function r(n){var e=n.attr("data-target"),e="#"!==(e=e||(e=n.attr("href"...
function i (line 1) | function i(l){l&&3===l.which||(a(".dropdown-backdrop").remove(),a(o).eac...
function r (line 1) | function r(n,e){this.options=e,this.$body=o(document.body),this.$element...
function i (line 1) | function i(l,a){return this.each(function(){var n=o(this),e=n.data("bs.m...
function o (line 1) | function o(n,e){this.init("popover",n,e)}
function l (line 1) | function l(n,e){this.$body=a(document.body),this.$scrollElement=a(n).is(...
function e (line 1) | function e(t){return this.each(function(){var n=a(this),e=n.data("bs.scr...
function i (line 1) | function i(n){this.element=r(n)}
function e (line 1) | function e(t){return this.each(function(){var n=r(this),e=n.data("bs.tab...
function o (line 1) | function o(){l.removeClass("active").find("> .dropdown-menu > .active")....
function t (line 1) | function t(n){n.preventDefault(),e.call(r(this),"show")}
function a (line 1) | function a(n,e,t){if(0===n.length)return n;if(t&&"function"==typeof t)re...
function s (line 1) | function s(n,e){this.type=null,this.options=null,this.enabled=null,this....
function a (line 1) | function a(){"in"!=e.hoverState&&t.detach(),e.$element&&e.$element.remov...
function l (line 1) | function l(n){return n&&n.__esModule?n:{default:n}}
function a (line 1) | function a(n){if(n&&n.__esModule)return n;var e={};if(null!=n)for(var t ...
function h (line 1) | function h(){var e=new o.HandlebarsEnvironment;return s.extend(e,o),e.Sa...
function l (line 1) | function l(n){return n&&n.__esModule?n:{default:n}}
function h (line 1) | function h(n,e,t){this.helpers=n||{},this.partials=e||{},this.decorators...
function c (line 1) | function c(n,e){var t=e&&e.loc,l=void 0,a=void 0,o=void 0,e=void 0;t&&(l...
function l (line 1) | function l(n){return n&&n.__esModule?n:{default:n}}
function c (line 1) | function c(n,e,t){i&&(i.key=n,i.index=e,i.first=0===e,i.last=!!t,s&&(i.c...
function r (line 1) | function r(n,e){return void 0!==n.whitelist[e]?!0===n.whitelist[e]:void ...
function i (line 1) | function i(n){var e=arguments.length<=1||void 0===arguments[1]?{}:argume...
function f (line 1) | function f(l,n,a,o,e,r,i){function t(n){var e=arguments.length<=1||void ...
function r (line 1) | function r(){return""}
function g (line 1) | function g(n,e,t,l,a,o){return n.decorator&&(e=n.decorator(e,n={},t,l&&l...
function l (line 1) | function l(n){this.string=n}
function r (line 1) | function r(n){return l[n]}
function i (line 1) | function i(n){for(var e=1;e<arguments.length;e++)for(var t in arguments[...
function u (line 1) | function u(n,e){var t=e;return"counter"===n?t=m.get("STATSD_COUNTER_FORM...
function h (line 1) | function h(n){return{depth:"gauge",in_flight_count:"gauge",deferred_coun...
function i (line 1) | function i(n,e){if("topic"===n||"channel"===n){if("depth"===e||"deferred...
function p (line 1) | function p(n){return n.replaceAll(" ","_").replaceAll("/","-").replaceAl...
function s (line 1) | function s(n,e,t,l,a){var o,r,i,s=[],c=(o=e||"*",r=m.get("STATSD_PREFIX"...
function a (line 1) | function a(n,e){e=Math.pow(10,e);return Math.round(n*e)/e}
function n (line 1) | function n(n){return a.basePath(n).substring(1)}
FILE: nsqadmin/static/build/vendor.js
function s (line 1) | function s(t,e){if(!o[t]){if(!i[t]){var n="function"==typeof require&&re...
function n (line 1) | function n(){r.stopListening(),r.trigger("destroy",r,r.collection,t)}
function w (line 1) | function w(e,t,n){n=Math.min(Math.max(n,0),e.length);for(var r=Array(e.l...
function S (line 1) | function S(n,r,e,i){x.each(e,function(e,t){r[t]&&(n.prototype[t]=functio...
function l (line 1) | function l(e,t,n){return c.extend(!0,{},e,function(e,t){var n=e.length,r...
function g (line 1) | function g(e,t,n,r){r&&r[0]&&(a=r[0].locale||h.locale,(r[0].swapButtonOr...
function v (line 1) | function v(e){return Object.keys(e).length}
function m (line 1) | function m(e,n){var r=0;c.each(e,function(e,t){n(e,t,r++)})}
function y (line 1) | function y(e){e.data.dialog.find(".bootbox-accept").first().trigger("foc...
function b (line 1) | function b(e){e.target===e.data.dialog[0]&&e.data.dialog.remove()}
function x (line 1) | function x(e){e.target===e.data.dialog[0]&&(e.data.dialog.off("escape.cl...
function w (line 1) | function w(e,t,n){e.stopPropagation(),e.preventDefault(),c.isFunction(n)...
function C (line 1) | function C(e){return/([01][0-9]|2[0-3]):[0-5][0-9]?:[0-5][0-9]/.test(e)}
function E (line 1) | function E(e){return/(\d{4})-(\d{2})-(\d{2})/.test(e)}
function g (line 1) | function g(e){return null!=e&&e===e.window}
function x (line 1) | function x(e,t,n){var r,i,o=(n=n||E).createElement("script");if(o.text=e...
function h (line 1) | function h(e){return null==e?e+"":"object"==typeof e||"function"==typeof...
function f (line 1) | function f(e){var t=!!e&&"length"in e&&e.length,n=h(e);return!b(e)&&!g(e...
function f (line 1) | function f(e,t){return e="0x"+e.slice(1)-65536,t||(e<0?String.fromCharCo...
function r (line 1) | function r(){C()}
function se (line 1) | function se(t,e,n,r){var i,o,a,s,u,l,c=e&&e.ownerDocument,f=e?e.nodeType...
function ue (line 1) | function ue(){var n=[];function r(e,t){return n.push(e+" ")>x.cacheLengt...
function le (line 1) | function le(e){return e[k]=!0,e}
function ce (line 1) | function ce(e){var t=E.createElement("fieldset");try{return!!e(t)}catch(...
function fe (line 1) | function fe(e,t){for(var n=e.split("|"),r=n.length;r--;)x.attrHandle[n[r...
function pe (line 1) | function pe(e,t){var n=t&&e,r=n&&1===e.nodeType&&1===t.nodeType&&e.sourc...
function de (line 1) | function de(t){return function(e){return"form"in e?e.parentNode&&!1===e....
function he (line 1) | function he(a){return le(function(o){return o=+o,le(function(e,t){for(va...
function ge (line 1) | function ge(e){return e&&void 0!==e.getElementsByTagName&&e}
function ve (line 1) | function ve(){}
function me (line 1) | function me(e){for(var t=0,n=e.length,r="";t<n;t++)r+=e[t].value;return r}
function ye (line 1) | function ye(a,e,t){var s=e.dir,u=e.next,l=u||s,c=t&&"parentNode"===l,f=b...
function be (line 1) | function be(i){return 1<i.length?function(e,t,n){for(var r=i.length;r--;...
function xe (line 1) | function xe(e,t,n,r,i){for(var o,a=[],s=0,u=e.length,l=null!=t;s<u;s++)(...
function we (line 1) | function we(d,h,g,v,m,e){return v&&!v[k]&&(v=we(v)),m&&!m[k]&&(m=we(m,e)...
function Ce (line 1) | function Ce(v,m){function e(e,t,n,r,i){var o,a,s,u=0,l="0",c=e&&[],f=[],...
function d (line 1) | function d(e,t,n){for(var r=[],i=void 0!==n;(e=e[t])&&9!==e.nodeType;)if...
function w (line 1) | function w(e,t){for(var n=[];e;e=e.nextSibling)1===e.nodeType&&e!==t&&n....
function N (line 1) | function N(e,t){return e.nodeName&&e.nodeName.toLowerCase()===t.toLowerC...
function O (line 1) | function O(e,n,r){return b(n)?T.grep(e,function(e,t){return!!n.call(e,t,...
function I (line 1) | function I(e,t){for(;(e=e[t])&&1!==e.nodeType;);return e}
function M (line 1) | function M(e){return e}
function q (line 1) | function q(e){throw e}
function R (line 1) | function R(e,t,n,r){var i;try{e&&b(i=e.promise)?i.call(e).done(t).fail(n...
function i (line 1) | function i(){for(a=a||r.once,t=o=!0;u.length;l=-1)for(e=u.shift();++l<s....
function l (line 1) | function l(i,o,a,s){return function(){function e(){var e,t;if(!(i<u)){if...
function t (line 1) | function t(t){return function(e){i[t]=this,o[t]=1<arguments.length?s.cal...
function F (line 1) | function F(){E.removeEventListener("DOMContentLoaded",F),C.removeEventLi...
function W (line 1) | function W(e,t){return t.toUpperCase()}
function z (line 1) | function z(e){return e.replace(K,"ms-").replace($,W)}
function U (line 1) | function U(e){return 1===e.nodeType||9===e.nodeType||!+e.nodeType}
function V (line 1) | function V(){this.expando=T.expando+V.uid++}
function Q (line 1) | function Q(e,t,n){var r,i;if(void 0===n&&1===e.nodeType)if(r="data-"+t.r...
function n (line 1) | function n(){--i||o.resolveWith(a,[a])}
function ae (line 1) | function ae(e,t,n,r){var i,o,a=20,s=r?function(){return r.cur()}:functio...
function ue (line 1) | function ue(e,t){for(var n,r,i,o,a,s=[],u=0,l=e.length;u<l;u++)(r=e[u])....
function de (line 1) | function de(e,t){var n=void 0!==e.getElementsByTagName?e.getElementsByTa...
function he (line 1) | function he(e,t){for(var n=0,r=e.length;n<r;n++)X.set(e[n],"globalEval",...
function ve (line 1) | function ve(e,t,n,r,i){for(var o,a,s,u,l,c=t.createDocumentFragment(),f=...
function ye (line 1) | function ye(){return!0}
function be (line 1) | function be(){return!1}
function xe (line 1) | function xe(e,t){return e===function(){try{return E.activeElement}catch(...
function we (line 1) | function we(e,t,n,r,i,o){var a,s;if("object"==typeof t){for(s in"string"...
function Ce (line 1) | function Ce(e,i,o){o?(X.set(e,i,!1),T.event.add(e,i,{namespace:!1,handle...
function Ne (line 1) | function Ne(e,t){return N(e,"table")&&N(11!==t.nodeType?t:t.firstChild,"...
function Ae (line 1) | function Ae(e){return e.type=(null!==e.getAttribute("type"))+"/"+e.type,e}
function Oe (line 1) | function Oe(e){return"true/"===(e.type||"").slice(0,5)?e.type=e.type.sli...
function Se (line 1) | function Se(e,t){var n,r,i,o;if(1===t.nodeType){if(X.hasData(e)&&(o=X.ge...
function je (line 1) | function je(n,r,i,o){r=v(r);var e,t,a,s,u,l,c=0,f=n.length,p=f-1,d=r[0],...
function _e (line 1) | function _e(e,t,n){for(var r,i=t?T.filter(t,e):e,o=0;null!=(r=i[o]);o++)...
function Le (line 1) | function Le(e,t,n){var r,i={};for(r in t)i[r]=e.style[r],e.style[r]=t[r]...
function We (line 1) | function We(){var e;Fe&&(Pe.style.cssText="position:absolute;left:-11111...
function ze (line 1) | function ze(e){return Math.round(parseFloat(e))}
function Ue (line 1) | function Ue(e,t,n){var r,i,o=e.style;return(n=n||Ke(e))&&(""!==(i=n.getP...
function Ve (line 1) | function Ve(e,t){return{get:function(){if(!e())return(this.get=t).apply(...
function Ge (line 1) | function Ge(e){var t=T.cssProps[e]||Je[e];return t||(e in Ye?e:Je[e]=fun...
function nt (line 1) | function nt(e,t,n){var r=ee.exec(t);return r?Math.max(0,r[2]-(n||0))+(r[...
function rt (line 1) | function rt(e,t,n,r,i,o){var a="width"===t?1:0,s=0,u=0;if(n===(r?"border...
function it (line 1) | function it(e,t,n){var r=Ke(e),i=(!y.boxSizingReliable()||n)&&"border-bo...
function ot (line 1) | function ot(e,t,n,r,i){return new ot.prototype.init(e,t,n,r,i)}
function ct (line 1) | function ct(){st&&(!1===E.hidden&&C.requestAnimationFrame?C.requestAnima...
function ft (line 1) | function ft(){return C.setTimeout(function(){at=void 0}),at=Date.now()}
function pt (line 1) | function pt(e,t){var n,r=0,i={height:e};for(t=t?1:0;r<4;r+=2-t)i["margin...
function dt (line 1) | function dt(e,t,n){for(var r,i=(ht.tweeners[t]||[]).concat(ht.tweeners["...
function ht (line 1) | function ht(i,e,t){var n,o,r=0,a=ht.prefilters.length,s=T.Deferred().alw...
function a (line 1) | function a(e){var t=e.stop;delete e.stop,t(o)}
function bt (line 1) | function bt(e){return(e.match(D)||[]).join(" ")}
function xt (line 1) | function xt(e){return e.getAttribute&&e.getAttribute("class")||""}
function wt (line 1) | function wt(e){return Array.isArray(e)?e:"string"==typeof e&&e.match(D)|...
function Et (line 1) | function Et(e){e.stopPropagation()}
function i (line 1) | function i(e){T.event.simulate(r,e.target,T.event.fix(e))}
function n (line 1) | function n(e,t){t=b(t)?t():t,i[i.length]=encodeURIComponent(e)+"="+encod...
function Kt (line 1) | function Kt(o){return function(e,t){"string"!=typeof e&&(t=e,e="*");var ...
function $t (line 1) | function $t(t,r,i,o){var a={},s=t===Pt;function u(e){var n;return a[e]=!...
function Wt (line 1) | function Wt(e,t){var n,r,i=T.ajaxSettings.flatOptions||{};for(n in t)voi...
function w (line 1) | function w(e,t,n,r){var i,o,a,s=t;p||(p=!0,f&&C.clearTimeout(f),u=void 0...
function b (line 1) | function b(i,o){return o=null==o?i.length-1:+o,function(){for(var e=Math...
function x (line 1) | function x(e){var t=typeof e;return"function"==t||"object"==t&&!!e}
function w (line 1) | function w(e){return void 0===e}
function C (line 1) | function C(e){return!0===e||!1===e||"[object Boolean]"===p.call(e)}
function E (line 1) | function E(e){var t="[object "+e+"]";return function(e){return p.call(e)...
function F (line 1) | function F(e,t){return null!=e&&n.call(e,t)}
function $ (line 1) | function $(e){return k(e)&&h(e)}
function W (line 1) | function W(e){return function(){return e}}
function z (line 1) | function z(t){return function(e){e=t(e);return"number"==typeof e&&0<=e&&...
function U (line 1) | function U(t){return function(e){return null==e?void 0:e[t]}}
function Q (line 1) | function Q(e,t){t=function(t){for(var n={},e=t.length,r=0;r<e;++r)n[t[r]...
function Z (line 1) | function Z(e){if(!x(e))return[];if(l)return l(e);var t,n=[];for(t in e)F...
function ee (line 1) | function ee(e,t){var n=Z(t),r=n.length;if(null==e)return!r;for(var i=Obj...
function te (line 1) | function te(e){return e instanceof te?e:this instanceof te?void(this._wr...
function ne (line 1) | function ne(e){return new Uint8Array(e.buffer||e,e.byteOffset||0,V(e))}
function ie (line 1) | function ie(e,t,n,r){if(e===t)return 0!==e||1/e==1/t;if(null==e||null==t...
function oe (line 1) | function oe(e){if(!x(e))return[];var t,n=[];for(t in e)n.push(t);return ...
function ae (line 1) | function ae(r){var i=G(r);return function(e){if(null==e)return!1;var t=o...
function me (line 1) | function me(e){for(var t=Z(e),n=t.length,r=Array(n),i=0;i<n;i++)r[i]=e[t...
function ye (line 1) | function ye(e){for(var t={},n=Z(e),r=0,i=n.length;r<i;r++)t[e[n[r]]]=n[r...
function be (line 1) | function be(e){var t,n=[];for(t in e)I(e[t])&&n.push(t);return n.sort()}
function xe (line 1) | function xe(u,l){return function(e){var t=arguments.length;if(l&&(e=Obje...
function Te (line 1) | function Te(e){if(!x(e))return{};if(c)return c(e);var t=function(){};t.p...
function ke (line 1) | function ke(e){return x(e)?P(e)?e.slice():we({},e):e}
function Ne (line 1) | function Ne(e){return P(e)?e:[e]}
function Ae (line 1) | function Ae(e){return te.toPath(e)}
function Oe (line 1) | function Oe(e,t){for(var n=t.length,r=0;r<n;r++){if(null==e)return;e=e[t...
function Se (line 1) | function Se(e,t,n){t=Oe(e,Ae(t));return w(t)?n:t}
function je (line 1) | function je(e){return e}
function _e (line 1) | function _e(t){return t=Ce({},t),function(e){return ee(e,t)}}
function Le (line 1) | function Le(t){return t=Ae(t),function(e){return Oe(e,t)}}
function Ie (line 1) | function Ie(i,o,e){if(void 0===o)return i;switch(null==e?3:e){case 1:ret...
function De (line 1) | function De(e,t,n){return null==e?je:I(e)?Ie(e,t,n):(x(e)&&!P(e)?_e:Le)(e)}
function Me (line 1) | function Me(e,t){return De(e,t,1/0)}
function qe (line 1) | function qe(e,t,n){return te.iteratee!==Me?te.iteratee(e,t):De(e,t,n)}
function Re (line 1) | function Re(){}
function He (line 1) | function He(e,t){return null==t&&(t=e,e=0),e+Math.floor(Math.random()*(t...
function Fe (line 1) | function Fe(t){function n(e){return t[e]}var e="(?:"+Z(t).join("|")+")",...
function Xe (line 1) | function Xe(e){return"\\"+Ue[e]}
function Ge (line 1) | function Ge(e,t,n,r,i){if(!(r instanceof t))return e.apply(n,i);n=Te(e.p...
function tt (line 1) | function tt(e,t,n,r){if(r=r||[],t||0===t){if(t<=0)return r.concat(e)}els...
function ot (line 1) | function ot(e){return function(){return!e.apply(this,arguments)}}
function at (line 1) | function at(e,t){var n;return function(){return 0<--e&&(n=t.apply(this,a...
function st (line 1) | function st(e,t,n){t=qe(t,n);for(var r,i=Z(e),o=0,a=i.length;o<a;o++)if(...
function ut (line 1) | function ut(o){return function(e,t,n){t=qe(t,n);for(var r=G(e),i=0<o?0:r...
function ct (line 1) | function ct(e,t,n,r){for(var i=(n=qe(n,r,1))(t),o=0,a=G(e);o<a;){var s=M...
function ft (line 1) | function ft(o,a,s){return function(e,t,n){var r=0,i=G(e);if("number"==ty...
function dt (line 1) | function dt(e,t,n){n=(et(e)?lt:st)(e,t,n);if(void 0!==n&&-1!==n)return e...
function ht (line 1) | function ht(e,t,n){if(t=Ie(t,n),et(e))for(i=0,o=e.length;i<o;i++)t(e[i],...
function gt (line 1) | function gt(e,t,n){t=qe(t,n);for(var r=!et(e)&&Z(e),i=(r||e).length,o=Ar...
function vt (line 1) | function vt(u){return function(e,t,n,r){var i=3<=arguments.length;return...
function mt (line 1) | function mt(e,r,t){var i=[];return r=qe(r,t),ht(e,function(e,t,n){r(e,t,...
function yt (line 1) | function yt(e,t,n){t=qe(t,n);for(var r=!et(e)&&Z(e),i=(r||e).length,o=0;...
function bt (line 1) | function bt(e,t,n){t=qe(t,n);for(var r=!et(e)&&Z(e),i=(r||e).length,o=0;...
function xt (line 1) | function xt(e,t,n,r){return et(e)||(e=me(e)),0<=pt(e,t,n="number"!=typeo...
function wt (line 1) | function wt(e,t){return gt(e,Le(t))}
function Ct (line 1) | function Ct(e,r,t){var n,i,o=-1/0,a=-1/0;if(null==r||"number"==typeof r&...
function Et (line 1) | function Et(e,t,n){if(null==t||n)return(e=!et(e)?me(e):e)[He(e.length-1)...
function Tt (line 1) | function Tt(o,t){return function(n,r,e){var i=t?[[],[]]:{};return r=qe(r...
function Nt (line 1) | function Nt(e,t,n){return t in n}
function Ot (line 1) | function Ot(e,t,n){return u.call(e,0,Math.max(0,e.length-(null==t||n?1:t...
function St (line 1) | function St(e,t,n){return null==e||e.length<1?null==t||n?void 0:[]:null=...
function jt (line 1) | function jt(e,t,n){return u.call(e,null==t||n?1:t)}
function Lt (line 1) | function Lt(e,t,n,r){C(t)||(r=n,n=t,t=!1),null!=n&&(n=qe(n,r));for(var i...
function It (line 1) | function It(e){for(var t=e&&Ct(e,G).length||0,n=Array(t),r=0;r<t;r++)n[r...
function Dt (line 1) | function Dt(e,t){return e._chain?te(t).chain():t}
function Mt (line 1) | function Mt(n){return ht(be(n),function(e){var t=te[e]=n[e];te.prototype...
function c (line 1) | function c(){l=!1===i.leading?0:Pe(),o=null,u=n.apply(a,s),o||(a=s=null)}
function e (line 1) | function e(){var e=Pe();l||!1!==i.leading||(l=e);var t=r-(e-l);return a=...
FILE: nsqadmin/static/js/lib/handlebars_helpers.js
function round (line 208) | function round(num, places) {
FILE: nsqd/backend_queue.go
type BackendQueue (line 5) | type BackendQueue interface
FILE: nsqd/buffer_pool.go
function init (line 10) | func init() {
function bufferPoolGet (line 16) | func bufferPoolGet() *bytes.Buffer {
function bufferPoolPut (line 20) | func bufferPoolPut(b *bytes.Buffer) {
FILE: nsqd/channel.go
type Consumer (line 20) | type Consumer interface
type Channel (line 37) | type Channel struct
method initPQ (line 141) | func (c *Channel) initPQ() {
method Exiting (line 156) | func (c *Channel) Exiting() bool {
method Delete (line 161) | func (c *Channel) Delete() error {
method Close (line 166) | func (c *Channel) Close() error {
method exit (line 170) | func (c *Channel) exit(deleted bool) error {
method Empty (line 206) | func (c *Channel) Empty() error {
method flush (line 231) | func (c *Channel) flush() error {
method Depth (line 282) | func (c *Channel) Depth() int64 {
method Pause (line 286) | func (c *Channel) Pause() error {
method UnPause (line 290) | func (c *Channel) UnPause() error {
method doPause (line 294) | func (c *Channel) doPause(pause bool) error {
method IsPaused (line 313) | func (c *Channel) IsPaused() bool {
method PutMessage (line 318) | func (c *Channel) PutMessage(m *Message) error {
method put (line 332) | func (c *Channel) put(m *Message) error {
method PutMessageDeferred (line 382) | func (c *Channel) PutMessageDeferred(msg *Message, timeout time.Durati...
method TouchMessage (line 388) | func (c *Channel) TouchMessage(clientID int64, id MessageID, clientMsg...
method FinishMessage (line 412) | func (c *Channel) FinishMessage(clientID int64, id MessageID) error {
method RequeueMessage (line 430) | func (c *Channel) RequeueMessage(clientID int64, id MessageID, timeout...
method AddClient (line 455) | func (c *Channel) AddClient(clientID int64, client Consumer) error {
method RemoveClient (line 484) | func (c *Channel) RemoveClient(clientID int64) {
method StartInFlightTimeout (line 509) | func (c *Channel) StartInFlightTimeout(msg *Message, clientID int64, t...
method StartDeferredTimeout (line 522) | func (c *Channel) StartDeferredTimeout(msg *Message, timeout time.Dura...
method pushInFlightMessage (line 534) | func (c *Channel) pushInFlightMessage(msg *Message) error {
method popInFlightMessage (line 547) | func (c *Channel) popInFlightMessage(clientID int64, id MessageID) (*M...
method addToInFlightPQ (line 563) | func (c *Channel) addToInFlightPQ(msg *Message) {
method removeFromInFlightPQ (line 569) | func (c *Channel) removeFromInFlightPQ(msg *Message) {
method pushDeferredMessage (line 580) | func (c *Channel) pushDeferredMessage(item *pqueue.Item) error {
method popDeferredMessage (line 594) | func (c *Channel) popDeferredMessage(id MessageID) (*pqueue.Item, erro...
method addToDeferredPQ (line 607) | func (c *Channel) addToDeferredPQ(item *pqueue.Item) {
method processDeferredQueue (line 613) | func (c *Channel) processDeferredQueue(t int64) bool {
method processInFlightQueue (line 644) | func (c *Channel) processInFlightQueue(t int64) bool {
function NewChannel (line 81) | func NewChannel(topicName string, channelName string, nsqd *NSQD,
FILE: nsqd/channel_test.go
function TestPutMessage (line 16) | func TestPutMessage(t *testing.T) {
function TestPutMessage2Chan (line 37) | func TestPutMessage2Chan(t *testing.T) {
function TestInFlightWorker (line 62) | func TestInFlightWorker(t *testing.T) {
function TestChannelEmpty (line 107) | func TestChannelEmpty(t *testing.T) {
function TestChannelEmptyConsumer (line 140) | func TestChannelEmptyConsumer(t *testing.T) {
function TestMaxChannelConsumers (line 177) | func TestMaxChannelConsumers(t *testing.T) {
function TestChannelHealth (line 203) | func TestChannelHealth(t *testing.T) {
FILE: nsqd/client_v2.go
constant defaultBufferSize (line 18) | defaultBufferSize = 16 * 1024
constant stateInit (line 21) | stateInit = iota
constant stateDisconnected (line 22) | stateDisconnected
constant stateConnected (line 23) | stateConnected
constant stateSubscribed (line 24) | stateSubscribed
constant stateClosing (line 25) | stateClosing
type identifyDataV2 (line 28) | type identifyDataV2 struct
type identifyEvent (line 46) | type identifyEvent struct
type PubCount (line 55) | type PubCount struct
type ClientV2Stats (line 60) | type ClientV2Stats struct
method String (line 94) | func (s ClientV2Stats) String() string {
type clientV2 (line 132) | type clientV2 struct
method String (line 240) | func (c *clientV2) String() string {
method Type (line 244) | func (c *clientV2) Type() int {
method Identify (line 254) | func (c *clientV2) Identify(data identifyDataV2) error {
method Stats (line 303) | func (c *clientV2) Stats(topicName string) ClientStats {
method IsReadyForMessages (line 417) | func (c *clientV2) IsReadyForMessages() bool {
method SetReadyCount (line 434) | func (c *clientV2) SetReadyCount(count int64) {
method tryUpdateReadyState (line 442) | func (c *clientV2) tryUpdateReadyState() {
method FinishedMessage (line 452) | func (c *clientV2) FinishedMessage() {
method Empty (line 458) | func (c *clientV2) Empty() {
method SendingMessage (line 463) | func (c *clientV2) SendingMessage() {
method PublishedMessage (line 468) | func (c *clientV2) PublishedMessage(topic string, count uint64) {
method TimedOutMessage (line 474) | func (c *clientV2) TimedOutMessage() {
method RequeuedMessage (line 479) | func (c *clientV2) RequeuedMessage() {
method StartClose (line 485) | func (c *clientV2) StartClose() {
method Pause (line 492) | func (c *clientV2) Pause() {
method UnPause (line 496) | func (c *clientV2) UnPause() {
method SetHeartbeatInterval (line 500) | func (c *clientV2) SetHeartbeatInterval(desiredInterval int) error {
method SetOutputBuffer (line 519) | func (c *clientV2) SetOutputBuffer(desiredSize int, desiredTimeout int...
method SetSampleRate (line 561) | func (c *clientV2) SetSampleRate(sampleRate int32) error {
method SetMsgTimeout (line 569) | func (c *clientV2) SetMsgTimeout(msgTimeout int) error {
method UpgradeTLS (line 586) | func (c *clientV2) UpgradeTLS() error {
method UpgradeDeflate (line 606) | func (c *clientV2) UpgradeDeflate(level int) error {
method UpgradeSnappy (line 626) | func (c *clientV2) UpgradeSnappy() error {
method Flush (line 644) | func (c *clientV2) Flush() error {
method QueryAuthd (line 664) | func (c *clientV2) QueryAuthd() error {
method Auth (line 697) | func (c *clientV2) Auth(secret string) error {
method IsAuthorized (line 702) | func (c *clientV2) IsAuthorized(topic, channel string) (bool, error) {
method HasAuthorizations (line 718) | func (c *clientV2) HasAuthorizations() bool {
function newClientV2 (line 198) | func newClientV2(id int64, conn net.Conn, nsqd *NSQD) *clientV2 {
type prettyConnectionState (line 366) | type prettyConnectionState struct
method GetCipherSuite (line 370) | func (p *prettyConnectionState) GetCipherSuite() string {
method GetVersion (line 402) | func (p *prettyConnectionState) GetVersion() string {
FILE: nsqd/dqname.go
function getBackendName (line 6) | func getBackendName(topicName, channelName string) string {
FILE: nsqd/dqname_windows.go
function getBackendName (line 7) | func getBackendName(topicName, channelName string) string {
FILE: nsqd/dummy_backend_queue.go
type dummyBackendQueue (line 3) | type dummyBackendQueue struct
method Put (line 11) | func (d *dummyBackendQueue) Put([]byte) error {
method ReadChan (line 15) | func (d *dummyBackendQueue) ReadChan() <-chan []byte {
method Close (line 19) | func (d *dummyBackendQueue) Close() error {
method Delete (line 23) | func (d *dummyBackendQueue) Delete() error {
method Depth (line 27) | func (d *dummyBackendQueue) Depth() int64 {
method Empty (line 31) | func (d *dummyBackendQueue) Empty() error {
function newDummyBackendQueue (line 7) | func newDummyBackendQueue() BackendQueue {
FILE: nsqd/guid.go
constant nodeIDBits (line 20) | nodeIDBits = uint64(10)
constant sequenceBits (line 21) | sequenceBits = uint64(12)
constant nodeIDShift (line 22) | nodeIDShift = sequenceBits
constant timestampShift (line 23) | timestampShift = sequenceBits + nodeIDBits
constant sequenceMask (line 24) | sequenceMask = int64(-1) ^ (int64(-1) << sequenceBits)
constant twepoch (line 27) | twepoch = int64(1288834974288)
type guid (line 34) | type guid
method Hex (line 90) | func (g guid) Hex() MessageID {
type guidFactory (line 36) | type guidFactory struct
method NewGUID (line 51) | func (f *guidFactory) NewGUID() (guid, error) {
function NewGUIDFactory (line 45) | func NewGUIDFactory(nodeID int64) *guidFactory {
FILE: nsqd/guid_test.go
function BenchmarkGUIDCopy (line 8) | func BenchmarkGUIDCopy(b *testing.B) {
function BenchmarkGUIDUnsafe (line 16) | func BenchmarkGUIDUnsafe(b *testing.B) {
function BenchmarkGUID (line 25) | func BenchmarkGUID(b *testing.B) {
FILE: nsqd/http.go
type httpServer (line 35) | type httpServer struct
method ServeHTTP (line 109) | func (s *httpServer) ServeHTTP(w http.ResponseWriter, req *http.Reques...
method pingHandler (line 122) | func (s *httpServer) pingHandler(w http.ResponseWriter, req *http.Requ...
method doInfo (line 130) | func (s *httpServer) doInfo(w http.ResponseWriter, req *http.Request, ...
method getExistingTopicFromQuery (line 173) | func (s *httpServer) getExistingTopicFromQuery(req *http.Request) (*ht...
method getTopicFromQuery (line 193) | func (s *httpServer) getTopicFromQuery(req *http.Request) (url.Values,...
method doPUB (line 213) | func (s *httpServer) doPUB(w http.ResponseWriter, req *http.Request, p...
method doMPUB (line 263) | func (s *httpServer) doMPUB(w http.ResponseWriter, req *http.Request, ...
method doCreateTopic (line 341) | func (s *httpServer) doCreateTopic(w http.ResponseWriter, req *http.Re...
method doEmptyTopic (line 346) | func (s *httpServer) doEmptyTopic(w http.ResponseWriter, req *http.Req...
method doDeleteTopic (line 375) | func (s *httpServer) doDeleteTopic(w http.ResponseWriter, req *http.Re...
method doPauseTopic (line 395) | func (s *httpServer) doPauseTopic(w http.ResponseWriter, req *http.Req...
method doCreateChannel (line 430) | func (s *httpServer) doCreateChannel(w http.ResponseWriter, req *http....
method doEmptyChannel (line 439) | func (s *httpServer) doEmptyChannel(w http.ResponseWriter, req *http.R...
method doDeleteChannel (line 458) | func (s *httpServer) doDeleteChannel(w http.ResponseWriter, req *http....
method doPauseChannel (line 472) | func (s *httpServer) doPauseChannel(w http.ResponseWriter, req *http.R...
method doStats (line 501) | func (s *httpServer) doStats(w http.ResponseWriter, req *http.Request,...
method printStats (line 548) | func (s *httpServer) printStats(stats Stats, ms *memStats, health stri...
method doConfig (line 628) | func (s *httpServer) doConfig(w http.ResponseWriter, req *http.Request...
function newHTTPServer (line 42) | func newHTTPServer(nsqd *NSQD, tlsEnabled bool, tlsRequired bool) *httpS...
function setBlockRateHandler (line 95) | func setBlockRateHandler(w http.ResponseWriter, req *http.Request, ps ht...
function freeMemory (line 104) | func freeMemory(w http.ResponseWriter, req *http.Request, ps httprouter....
function getOptByCfgName (line 672) | func getOptByCfgName(opts interface{}, name string) (interface{}, bool) {
FILE: nsqd/http_test.go
type ErrMessage (line 26) | type ErrMessage struct
type InfoDoc (line 30) | type InfoDoc struct
function TestHTTPpub (line 39) | func TestHTTPpub(t *testing.T) {
function TestHTTPpubEmpty (line 62) | func TestHTTPpubEmpty(t *testing.T) {
function TestHTTPmpub (line 86) | func TestHTTPmpub(t *testing.T) {
function TestHTTPmpubEmpty (line 115) | func TestHTTPmpubEmpty(t *testing.T) {
function TestHTTPmpubBinary (line 146) | func TestHTTPmpubBinary(t *testing.T) {
function TestHTTPmpubForNonNormalizedBinaryParam (line 175) | func TestHTTPmpubForNonNormalizedBinaryParam(t *testing.T) {
function TestHTTPpubDefer (line 204) | func TestHTTPpubDefer(t *testing.T) {
function TestHTTPSRequire (line 231) | func TestHTTPSRequire(t *testing.T) {
function TestHTTPSRequireVerify (line 276) | func TestHTTPSRequireVerify(t *testing.T) {
function TestTLSRequireVerifyExceptHTTP (line 340) | func TestTLSRequireVerifyExceptHTTP(t *testing.T) {
function TestHTTPV1TopicChannel (line 370) | func TestHTTPV1TopicChannel(t *testing.T) {
function TestHTTPClientStats (line 503) | func TestHTTPClientStats(t *testing.T) {
function TestHTTPgetStatusJSON (line 566) | func TestHTTPgetStatusJSON(t *testing.T) {
function TestHTTPgetStatusText (line 586) | func TestHTTPgetStatusText(t *testing.T) {
function TestHTTPconfig (line 605) | func TestHTTPconfig(t *testing.T) {
function TestHTTPerrors (line 662) | func TestHTTPerrors(t *testing.T) {
function TestDeleteTopic (line 686) | func TestDeleteTopic(t *testing.T) {
function TestEmptyTopic (line 735) | func TestEmptyTopic(t *testing.T) {
function TestEmptyChannel (line 797) | func TestEmptyChannel(t *testing.T) {
function TestInfo (line 876) | func TestInfo(t *testing.T) {
function BenchmarkHTTPpub (line 898) | func BenchmarkHTTPpub(b *testing.B) {
FILE: nsqd/in_flight_pqueue.go
type inFlightPqueue (line 3) | type inFlightPqueue
method Swap (line 9) | func (pq inFlightPqueue) Swap(i, j int) {
method Push (line 15) | func (pq *inFlightPqueue) Push(x *Message) {
method Pop (line 29) | func (pq *inFlightPqueue) Pop() *Message {
method Remove (line 45) | func (pq *inFlightPqueue) Remove(i int) *Message {
method PeekAndShift (line 58) | func (pq *inFlightPqueue) PeekAndShift(max int64) (*Message, int64) {
method up (line 72) | func (pq *inFlightPqueue) up(j int) {
method down (line 83) | func (pq *inFlightPqueue) down(i, n int) {
function newInFlightPqueue (line 5) | func newInFlightPqueue(capacity int) inFlightPqueue {
FILE: nsqd/in_flight_pqueue_test.go
function TestPriorityQueue (line 12) | func TestPriorityQueue(t *testing.T) {
function TestUnsortedInsert (line 29) | func TestUnsortedInsert(t *testing.T) {
function TestRemove (line 50) | func TestRemove(t *testing.T) {
FILE: nsqd/logger.go
type Logger (line 7) | type Logger
constant LOG_DEBUG (line 10) | LOG_DEBUG = lg.DEBUG
constant LOG_INFO (line 11) | LOG_INFO = lg.INFO
constant LOG_WARN (line 12) | LOG_WARN = lg.WARN
constant LOG_ERROR (line 13) | LOG_ERROR = lg.ERROR
constant LOG_FATAL (line 14) | LOG_FATAL = lg.FATAL
method logf (line 17) | func (n *NSQD) logf(level lg.LogLevel, f string, args ...interface{}) {
FILE: nsqd/lookup.go
function connectCallback (line 15) | func connectCallback(n *NSQD, hostname string) func(*lookupPeer) {
method lookupLoop (line 80) | func (n *NSQD) lookupLoop() {
function in (line 178) | func in(s string, lst []string) bool {
method lookupdHTTPAddrs (line 187) | func (n *NSQD) lookupdHTTPAddrs() []string {
FILE: nsqd/lookup_peer.go
type lookupPeer (line 19) | type lookupPeer struct
method Connect (line 51) | func (lp *lookupPeer) Connect() error {
method String (line 62) | func (lp *lookupPeer) String() string {
method Read (line 67) | func (lp *lookupPeer) Read(data []byte) (int, error) {
method Write (line 73) | func (lp *lookupPeer) Write(data []byte) (int, error) {
method Close (line 79) | func (lp *lookupPeer) Close() error {
method Command (line 93) | func (lp *lookupPeer) Command(cmd *nsq.Command) ([]byte, error) {
type peerInfo (line 30) | type peerInfo struct
function newLookupPeer (line 40) | func newLookupPeer(addr string, maxBodySize int64, l lg.AppLogFunc, conn...
function readResponseBounded (line 129) | func readResponseBounded(r io.Reader, limit int64) ([]byte, error) {
FILE: nsqd/message.go
constant MsgIDLength (line 11) | MsgIDLength = 16
constant minValidMsgLength (line 12) | minValidMsgLength = MsgIDLength + 8 + 2
type MessageID (line 15) | type MessageID
type Message (line 17) | type Message struct
method WriteTo (line 39) | func (m *Message) WriteTo(w io.Writer) (int64, error) {
function NewMessage (line 31) | func NewMessage(id MessageID, body []byte) *Message {
function decodeMessage (line 77) | func decodeMessage(b []byte) (*Message, error) {
function writeMessageToBackend (line 92) | func writeMessageToBackend(msg *Message, bq BackendQueue) error {
FILE: nsqd/nsqd.go
constant TLSNotRequired (line 30) | TLSNotRequired = iota
constant TLSRequiredExceptHTTP (line 31) | TLSRequiredExceptHTTP
constant TLSRequired (line 32) | TLSRequired
type errStore (line 35) | type errStore struct
type NSQD (line 39) | type NSQD struct
method getOpts (line 195) | func (n *NSQD) getOpts() *Options {
method swapOpts (line 199) | func (n *NSQD) swapOpts(opts *Options) {
method triggerOptsNotification (line 203) | func (n *NSQD) triggerOptsNotification() {
method RealTCPAddr (line 210) | func (n *NSQD) RealTCPAddr() net.Addr {
method RealHTTPAddr (line 218) | func (n *NSQD) RealHTTPAddr() net.Addr {
method RealHTTPSAddr (line 225) | func (n *NSQD) RealHTTPSAddr() *net.TCPAddr {
method SetHealth (line 232) | func (n *NSQD) SetHealth(err error) {
method IsHealthy (line 236) | func (n *NSQD) IsHealthy() bool {
method GetError (line 240) | func (n *NSQD) GetError() error {
method GetHealth (line 245) | func (n *NSQD) GetHealth() string {
method GetStartTime (line 253) | func (n *NSQD) GetStartTime() time.Time {
method Main (line 257) | func (n *NSQD) Main() error {
method LoadMetadata (line 342) | func (n *NSQD) LoadMetadata() error {
method GetMetadata (line 389) | func (n *NSQD) GetMetadata(ephemeral bool) *Metadata {
method PersistMetadata (line 417) | func (n *NSQD) PersistMetadata() error {
method Exit (line 442) | func (n *NSQD) Exit() {
method GetTopic (line 484) | func (n *NSQD) GetTopic(topicName string) *Topic {
method GetExistingTopic (line 541) | func (n *NSQD) GetExistingTopic(topicName string) (*Topic, error) {
method DeleteExistingTopic (line 552) | func (n *NSQD) DeleteExistingTopic(topicName string) error {
method Notify (line 576) | func (n *NSQD) Notify(v interface{}, persist bool) {
method channels (line 601) | func (n *NSQD) channels() []*Channel {
method resizePool (line 618) | func (n *NSQD) resizePool(num int, workCh chan *Channel, responseCh ch...
method queueScanWorker (line 644) | func (n *NSQD) queueScanWorker(workCh chan *Channel, responseCh chan b...
method queueScanLoop (line 676) | func (n *NSQD) queueScanLoop() {
method IsAuthEnabled (line 793) | func (n *NSQD) IsAuthEnabled() bool {
method Context (line 798) | func (n *NSQD) Context() context.Context {
function New (line 77) | func New(opts *Options) (*NSQD, error) {
type Metadata (line 296) | type Metadata struct
type TopicMetadata (line 302) | type TopicMetadata struct
type ChannelMetadata (line 309) | type ChannelMetadata struct
function newMetadataFile (line 314) | func newMetadataFile(opts *Options) string {
function readOrEmpty (line 318) | func readOrEmpty(fn string) ([]byte, error) {
function writeSyncFile (line 328) | func writeSyncFile(fn string, data []byte) error {
function buildTLSConfig (line 730) | func buildTLSConfig(opts *Options) (*tls.Config, error) {
function buildClientTLSConfig (line 773) | func buildClientTLSConfig(opts *Options) (*tls.Config, error) {
FILE: nsqd/nsqd_test.go
constant ConnectTimeout (line 21) | ConnectTimeout = 2 * time.Second
constant RequestTimeout (line 22) | RequestTimeout = 5 * time.Second
function getMetadata (line 25) | func getMetadata(n *NSQD) (*Metadata, error) {
function TestStartup (line 40) | func TestStartup(t *testing.T) {
function TestEphemeralTopicsAndChannels (line 160) | func TestEphemeralTopicsAndChannels(t *testing.T) {
function TestPauseMetadata (line 208) | func TestPauseMetadata(t *testing.T) {
function mustStartNSQLookupd (line 243) | func mustStartNSQLookupd(opts *nsqlookupd.Options) (net.Addr, net.Addr, ...
function TestReconfigure (line 259) | func TestReconfigure(t *testing.T) {
function TestCluster (line 319) | func TestCluster(t *testing.T) {
function TestSetHealth (line 418) | func TestSetHealth(t *testing.T) {
function TestUnixSocketStartup (line 443) | func TestUnixSocketStartup(t *testing.T) {
FILE: nsqd/options.go
type Options (line 15) | type Options struct
method HasExperiment (line 106) | func (o Options) HasExperiment(e Experiment) bool {
type Experiment (line 96) | type Experiment
constant TopologyAwareConsumption (line 99) | TopologyAwareConsumption Experiment = "topology-aware-consumption"
function NewOptions (line 115) | func NewOptions() *Options {
FILE: nsqd/protocol_v2.go
constant frameTypeResponse (line 21) | frameTypeResponse int32 = 0
constant frameTypeError (line 22) | frameTypeError int32 = 1
constant frameTypeMessage (line 23) | frameTypeMessage int32 = 2
type protocolV2 (line 30) | type protocolV2 struct
method NewClient (line 34) | func (p *protocolV2) NewClient(conn net.Conn) protocol.Client {
method IOLoop (line 39) | func (p *protocolV2) IOLoop(c protocol.Client) error {
method SendMessage (line 124) | func (p *protocolV2) SendMessage(client *clientV2, msg *Message) error {
method Send (line 143) | func (p *protocolV2) Send(client *clientV2, frameType int32, data []by...
method Exec (line 168) | func (p *protocolV2) Exec(client *clientV2, params [][]byte) ([]byte, ...
method messagePump (line 203) | func (p *protocolV2) messagePump(client *clientV2, startedChan chan bo...
method IDENTIFY (line 380) | func (p *protocolV2) IDENTIFY(client *clientV2, params [][]byte) ([]by...
method AUTH (line 526) | func (p *protocolV2) AUTH(client *clientV2, params [][]byte) ([]byte, ...
method CheckAuth (line 596) | func (p *protocolV2) CheckAuth(client *clientV2, cmd, topicName, chann...
method SUB (line 618) | func (p *protocolV2) SUB(client *clientV2, params [][]byte) ([]byte, e...
method RDY (line 676) | func (p *protocolV2) RDY(client *clientV2, params [][]byte) ([]byte, e...
method FIN (line 713) | func (p *protocolV2) FIN(client *clientV2, params [][]byte) ([]byte, e...
method REQ (line 739) | func (p *protocolV2) REQ(client *clientV2, params [][]byte) ([]byte, e...
method CLS (line 786) | func (p *protocolV2) CLS(client *clientV2, params [][]byte) ([]byte, e...
method NOP (line 796) | func (p *protocolV2) NOP(client *clientV2, params [][]byte) ([]byte, e...
method PUB (line 800) | func (p *protocolV2) PUB(client *clientV2, params [][]byte) ([]byte, e...
method MPUB (line 850) | func (p *protocolV2) MPUB(client *clientV2, params [][]byte) ([]byte, ...
method DPUB (line 903) | func (p *protocolV2) DPUB(client *clientV2, params [][]byte) ([]byte, ...
method TOUCH (line 967) | func (p *protocolV2) TOUCH(client *clientV2, params [][]byte) ([]byte,...
function readMPUB (line 994) | func readMPUB(r io.Reader, tmp []byte, topic *Topic, maxMessageSize int6...
function getMessageID (line 1038) | func getMessageID(p []byte) (*MessageID, error) {
function readLen (line 1045) | func readLen(r io.Reader, tmp []byte) (int32, error) {
function enforceTLSPolicy (line 1053) | func enforceTLSPolicy(client *clientV2, p *protocolV2, command []byte) e...
FILE: nsqd/protocol_v2_test.go
function mustStartNSQD (line 33) | func mustStartNSQD(opts *Options) (net.Addr, net.Addr, *NSQD) {
function mustConnectNSQD (line 57) | func mustConnectNSQD(tcpAddr net.Addr) (net.Conn, error) {
function identify (line 66) | func identify(t *testing.T, conn io.ReadWriter, extra map[string]interfa...
function sub (line 84) | func sub(t *testing.T, conn io.ReadWriter, topicName string, channelName...
function authCmd (line 90) | func authCmd(t *testing.T, conn io.ReadWriter, authSecret string, expect...
function subFail (line 99) | func subFail(t *testing.T, conn io.ReadWriter, topicName string, channel...
function readValidate (line 107) | func readValidate(t *testing.T, conn io.Reader, f int32, d string) []byte {
function TestChannelTopicNames (line 118) | func TestChannelTopicNames(t *testing.T) {
function TestBasicV2 (line 129) | func TestBasicV2(t *testing.T) {
function TestMultipleConsumerV2 (line 162) | func TestMultipleConsumerV2(t *testing.T) {
function TestSameZoneConsumerV2 (line 213) | func TestSameZoneConsumerV2(t *testing.T) {
function TestClientTimeout (line 286) | func TestClientTimeout(t *testing.T) {
function TestClientHeartbeat (line 323) | func TestClientHeartbeat(t *testing.T) {
function TestClientHeartbeatDisableSUB (line 359) | func TestClientHeartbeatDisableSUB(t *testing.T) {
function TestClientHeartbeatDisable (line 380) | func TestClientHeartbeatDisable(t *testing.T) {
function TestMaxHeartbeatIntervalValid (line 402) | func TestMaxHeartbeatIntervalValid(t *testing.T) {
function TestMaxHeartbeatIntervalInvalid (line 420) | func TestMaxHeartbeatIntervalInvalid(t *testing.T) {
function TestPausing (line 439) | func TestPausing(t *testing.T) {
function TestEmptyCommand (line 505) | func TestEmptyCommand(t *testing.T) {
function TestSizeLimits (line 522) | func TestSizeLimits(t *testing.T) {
function TestDPUB (line 639) | func TestDPUB(t *testing.T) {
function TestTouch (line 682) | func TestTouch(t *testing.T) {
function TestMaxRdyCount (line 728) | func TestMaxRdyCount(t *testing.T) {
function TestFatalError (line 776) | func TestFatalError(t *testing.T) {
function TestOutputBuffering (line 800) | func TestOutputBuffering(t *testing.T) {
function TestOutputBufferingValidity (line 852) | func TestOutputBufferingValidity(t *testing.T) {
function TestTLS (line 895) | func TestTLS(t *testing.T) {
function TestTLSRequired (line 934) | func TestTLSRequired(t *testing.T) {
function TestTLSAuthRequire (line 983) | func TestTLSAuthRequire(t *testing.T) {
function TestTLSAuthRequireVerify (line 1049) | func TestTLSAuthRequireVerify(t *testing.T) {
function TestDeflate (line 1138) | func TestDeflate(t *testing.T) {
type readWriter (line 1169) | type readWriter struct
function TestSnappy (line 1174) | func TestSnappy(t *testing.T) {
function TestTLSDeflate (line 1228) | func TestTLSDeflate(t *testing.T) {
function TestSampling (line 1279) | func TestSampling(t *testing.T) {
function TestTLSSnappy (line 1352) | func TestTLSSnappy(t *testing.T) {
function TestClientMsgTimeout (line 1403) | func TestClientMsgTimeout(t *testing.T) {
function TestBadFin (line 1462) | func TestBadFin(t *testing.T) {
function TestReqTimeoutRange (line 1488) | func TestReqTimeoutRange(t *testing.T) {
function TestClientAuth (line 1548) | func TestClientAuth(t *testing.T) {
function runAuthTest (line 1577) | func runAuthTest(t *testing.T, authResponse string, authSecret string, a...
function TestIOLoopReturnsClientErrWhenSendFails (line 1672) | func TestIOLoopReturnsClientErrWhenSendFails(t *testing.T) {
function TestIOLoopReturnsClientErrWhenSendSucceeds (line 1681) | func TestIOLoopReturnsClientErrWhenSendSucceeds(t *testing.T) {
function testIOLoopReturnsClientErr (line 1690) | func testIOLoopReturnsClientErr(t *testing.T, fakeConn test.FakeNetConn) {
function BenchmarkProtocolV2Exec (line 1711) | func BenchmarkProtocolV2Exec(b *testing.B) {
function benchmarkProtocolV2PubMultiTopic (line 1726) | func benchmarkProtocolV2PubMultiTopic(b *testing.B, numTopics int) {
function BenchmarkProtocolV2PubMultiTopic1 (line 1794) | func BenchmarkProtocolV2PubMultiTopic1(b *testing.B) { benchmarkProtoco...
function BenchmarkProtocolV2PubMultiTopic2 (line 1795) | func BenchmarkProtocolV2PubMultiTopic2(b *testing.B) { benchmarkProtoco...
function BenchmarkProtocolV2PubMultiTopic4 (line 1796) | func BenchmarkProtocolV2PubMultiTopic4(b *testing.B) { benchmarkProtoco...
function BenchmarkProtocolV2PubMultiTopic8 (line 1797) | func BenchmarkProtocolV2PubMultiTopic8(b *testing.B) { benchmarkProtoco...
function BenchmarkProtocolV2PubMultiTopic16 (line 1798) | func BenchmarkProtocolV2PubMultiTopic16(b *testing.B) { benchmarkProtoco...
function BenchmarkProtocolV2PubMultiTopic32 (line 1799) | func BenchmarkProtocolV2PubMultiTopic32(b *testing.B) { benchmarkProtoco...
function benchmarkProtocolV2Pub (line 1801) | func benchmarkProtocolV2Pub(b *testing.B, size int) {
function BenchmarkProtocolV2Pub256 (line 1868) | func BenchmarkProtocolV2Pub256(b *testing.B) { benchmarkProtocolV2Pub(b...
function BenchmarkProtocolV2Pub512 (line 1869) | func BenchmarkProtocolV2Pub512(b *testing.B) { benchmarkProtocolV2Pub(b...
function BenchmarkProtocolV2Pub1k (line 1870) | func BenchmarkProtocolV2Pub1k(b *testing.B) { benchmarkProtocolV2Pub(b...
function BenchmarkProtocolV2Pub2k (line 1871) | func BenchmarkProtocolV2Pub2k(b *testing.B) { benchmarkProtocolV2Pub(b...
function BenchmarkProtocolV2Pub4k (line 1872) | func BenchmarkProtocolV2Pub4k(b *testing.B) { benchmarkProtocolV2Pub(b...
function BenchmarkProtocolV2Pub8k (line 1873) | func BenchmarkProtocolV2Pub8k(b *testing.B) { benchmarkProtocolV2Pub(b...
function BenchmarkProtocolV2Pub16k (line 1874) | func BenchmarkProtocolV2Pub16k(b *testing.B) { benchmarkProtocolV2Pub(b...
function BenchmarkProtocolV2Pub32k (line 1875) | func BenchmarkProtocolV2Pub32k(b *testing.B) { benchmarkProtocolV2Pub(b...
function BenchmarkProtocolV2Pub64k (line 1876) | func BenchmarkProtocolV2Pub64k(b *testing.B) { benchmarkProtocolV2Pub(b...
function BenchmarkProtocolV2Pub128k (line 1877) | func BenchmarkProtocolV2Pub128k(b *testing.B) { benchmarkProtocolV2Pub(b...
function BenchmarkProtocolV2Pub256k (line 1878) | func BenchmarkProtocolV2Pub256k(b *testing.B) { benchmarkProtocolV2Pub(b...
function BenchmarkProtocolV2Pub512k (line 1879) | func BenchmarkProtocolV2Pub512k(b *testing.B) { benchmarkProtocolV2Pub(b...
function BenchmarkProtocolV2Pub1m (line 1880) | func BenchmarkProtocolV2Pub1m(b *testing.B) { benchmarkProtocolV2Pub(b...
function benchmarkProtocolV2Sub (line 1882) | func benchmarkProtocolV2Sub(b *testing.B, size int) {
function subWorker (line 1919) | func subWorker(n int, workers int, tcpAddr net.Addr, topicName string, r...
function BenchmarkProtocolV2Sub256 (line 1961) | func BenchmarkProtocolV2Sub256(b *testing.B) { benchmarkProtocolV2Sub(b...
function BenchmarkProtocolV2Sub512 (line 1962) | func BenchmarkProtocolV2Sub512(b *testing.B) { benchmarkProtocolV2Sub(b...
function BenchmarkProtocolV2Sub1k (line 1963) | func BenchmarkProtocolV2Sub1k(b *testing.B) { benchmarkProtocolV2Sub(b...
function BenchmarkProtocolV2Sub2k (line 1964) | func BenchmarkProtocolV2Sub2k(b *testing.B) { benchmarkProtocolV2Sub(b...
function BenchmarkProtocolV2Sub4k (line 1965) | func BenchmarkProtocolV2Sub4k(b *testing.B) { benchmarkProtocolV2Sub(b...
function BenchmarkProtocolV2Sub8k (line 1966) | func BenchmarkProtocolV2Sub8k(b *testing.B) { benchmarkProtocolV2Sub(b...
function BenchmarkProtocolV2Sub16k (line 1967) | func BenchmarkProtocolV2Sub16k(b *testing.B) { benchmarkProtocolV2Sub(b...
function BenchmarkProtocolV2Sub32k (line 1968) | func BenchmarkProtocolV2Sub32k(b *testing.B) { benchmarkProtocolV2Sub(b...
function BenchmarkProtocolV2Sub64k (line 1969) | func BenchmarkProtocolV2Sub64k(b *testing.B) { benchmarkProtocolV2Sub(b...
function BenchmarkProtocolV2Sub128k (line 1970) | func BenchmarkProtocolV2Sub128k(b *testing.B) { benchmarkProtocolV2Sub(b...
function BenchmarkProtocolV2Sub256k (line 1971) | func BenchmarkProtocolV2Sub256k(b *testing.B) { benchmarkProtocolV2Sub(b...
function BenchmarkProtocolV2Sub512k (line 1972) | func BenchmarkProtocolV2Sub512k(b *testing.B) { benchmarkProtocolV2Sub(b...
function BenchmarkProtocolV2Sub1m (line 1973) | func BenchmarkProtocolV2Sub1m(b *testing.B) { benchmarkProtocolV2Sub(b...
function benchmarkProtocolV2MultiSub (line 1975) | func benchmarkProtocolV2MultiSub(b *testing.B, num int) {
function BenchmarkProtocolV2MultiSub1 (line 2017) | func BenchmarkProtocolV2MultiSub1(b *testing.B) { benchmarkProtocolV2Mu...
function BenchmarkProtocolV2MultiSub2 (line 2018) | func BenchmarkProtocolV2MultiSub2(b *testing.B) { benchmarkProtocolV2Mu...
function BenchmarkProtocolV2MultiSub4 (line 2019) | func BenchmarkProtocolV2MultiSub4(b *testing.B) { benchmarkProtocolV2Mu...
function BenchmarkProtocolV2MultiSub8 (line 2020) | func BenchmarkProtocolV2MultiSub8(b *testing.B) { benchmarkProtocolV2Mu...
function BenchmarkProtocolV2MultiSub16 (line 2021) | func BenchmarkProtocolV2MultiSub16(b *testing.B) { benchmarkProtocolV2Mu...
FILE: nsqd/protocol_v2_unixsocket_test.go
function mustUnixSocketStartNSQD (line 29) | func mustUnixSocketStartNSQD(opts *Options) (net.Addr, net.Addr, *NSQD) {
function mustUnixSocketConnectNSQD (line 54) | func mustUnixSocketConnectNSQD(addr net.Addr) (net.Conn, error) {
function TestUnixSocketBasicV2 (line 64) | func TestUnixSocketBasicV2(t *testing.T) {
function TestUnixSocketMultipleConsumerV2 (line 97) | func TestUnixSocketMultipleConsumerV2(t *testing.T) {
function TestUnixSocketClientTimeout (line 146) | func TestUnixSocketClientTimeout(t *testing.T) {
function TestUnixSocketClientHeartbeat (line 183) | func TestUnixSocketClientHeartbeat(t *testing.T) {
function TestUnixSocketClientHeartbeatDisableSUB (line 219) | func TestUnixSocketClientHeartbeatDisableSUB(t *testing.T) {
function TestUnixSocketClientHeartbeatDisable (line 240) | func TestUnixSocketClientHeartbeatDisable(t *testing.T) {
function TestUnixSocketMaxHeartbeatIntervalValid (line 262) | func TestUnixSocketMaxHeartbeatIntervalValid(t *testing.T) {
function TestUnixSocketMaxHeartbeatIntervalInvalid (line 280) | func TestUnixSocketMaxHeartbeatIntervalInvalid(t *testing.T) {
function TestUnixSocketPausing (line 299) | func TestUnixSocketPausing(t *testing.T) {
function TestUnixSocketEmptyCommand (line 365) | func TestUnixSocketEmptyCommand(t *testing.T) {
function TestUnixSocketSizeLimits (line 382) | func TestUnixSocketSizeLimits(t *testing.T) {
function TestUnixSocketDPUB (line 499) | func TestUnixSocketDPUB(t *testing.T) {
function TestUnixSocketTouch (line 542) | func TestUnixSocketTouch(t *testing.T) {
function TestUnixSocketMaxRdyCount (line 588) | func TestUnixSocketMaxRdyCount(t *testing.T) {
function TestUnixSocketFatalError (line 636) | func TestUnixSocketFatalError(t *testing.T) {
function TestUnixSocketOutputBuffering (line 660) | func TestUnixSocketOutputBuffering(t *testing.T) {
function TestUnixSocketOutputBufferingValidity (line 712) | func TestUnixSocketOutputBufferingValidity(t *testing.T) {
function TestUnixSocketTLS (line 755) | func TestUnixSocketTLS(t *testing.T) {
function TestUnixSocketTLSRequired (line 794) | func TestUnixSocketTLSRequired(t *testing.T) {
function TestUnixSocketTLSAuthRequire (line 843) | func TestUnixSocketTLSAuthRequire(t *testing.T) {
function TestUnixSocketTLSAuthRequireVerify (line 909) | func TestUnixSocketTLSAuthRequireVerify(t *testing.T) {
function TestUnixSocketDeflate (line 998) | func TestUnixSocketDeflate(t *testing.T) {
function TestUnixSocketSnappy (line 1029) | func TestUnixSocketSnappy(t *testing.T) {
function TestUnixSocketTLSDeflate (line 1083) | func TestUnixSocketTLSDeflate(t *testing.T) {
function TestUnixSocketSampling (line 1134) | func TestUnixSocketSampling(t *testing.T) {
function TestUnixSocketTLSSnappy (line 1207) | func TestUnixSocketTLSSnappy(t *testing.T) {
function TestUnixSocketClientMsgTimeout (line 1258) | func TestUnixSocketClientMsgTimeout(t *testing.T) {
function TestUnixSocketBadFin (line 1317) | func TestUnixSocketBadFin(t *testing.T) {
function TestUnixSocketReqTimeoutRange (line 1343) | func TestUnixSocketReqTimeoutRange(t *testing.T) {
function TestUnixSocketIOLoopReturnsClientErrWhenSendFails (line 1403) | func TestUnixSocketIOLoopReturnsClientErrWhenSendFails(t *testing.T) {
function TestUnixSocketIOLoopReturnsClientErrWhenSendSucceeds (line 1412) | func TestUnixSocketIOLoopReturnsClientErrWhenSendSucceeds(t *testing.T) {
function testUnixSocketIOLoopReturnsClientErr (line 1421) | func testUnixSocketIOLoopReturnsClientErr(t *testing.T, fakeConn test.Fa...
function BenchmarkUnixSocketProtocolV2Exec (line 1442) | func BenchmarkUnixSocketProtocolV2Exec(b *testing.B) {
function benchmarkUnixSocketProtocolV2PubMultiTopic (line 1457) | func benchmarkUnixSocketProtocolV2PubMultiTopic(b *testing.B, numTopics ...
function BenchmarkUnixSocketProtocolV2PubMultiTopic1 (line 1525) | func BenchmarkUnixSocketProtocolV2PubMultiTopic1(b *testing.B) {
function BenchmarkUnixSocketkProtocolV2PubMultiTopic2 (line 1528) | func BenchmarkUnixSocketkProtocolV2PubMultiTopic2(b *testing.B) {
function BenchmarkUnixSocketProtocolV2PubMultiTopic4 (line 1531) | func BenchmarkUnixSocketProtocolV2PubMultiTopic4(b *testing.B) {
function BenchmarkUnixSocketProtocolV2PubMultiTopic8 (line 1534) | func BenchmarkUnixSocketProtocolV2PubMultiTopic8(b *testing.B) {
function BenchmarkUnixSocketProtocolV2PubMultiTopic16 (line 1537) | func BenchmarkUnixSocketProtocolV2PubMultiTopic16(b *testing.B) {
function BenchmarkUnixSocketProtocolV2PubMultiTopic32 (line 1540) | func BenchmarkUnixSocketProtocolV2PubMultiTopic32(b *testing.B) {
function benchmarkUnixSocketProtocolV2Pub (line 1544) | func benchmarkUnixSocketProtocolV2Pub(b *testing.B, size int) {
function BenchmarkUnixSocketProtocolV2Pub256 (line 1611) | func BenchmarkUnixSocketProtocolV2Pub256(b *testing.B) { benchmarkUnixSo...
function BenchmarkUnixSocketProtocolV2Pub512 (line 1612) | func BenchmarkUnixSocketProtocolV2Pub512(b *testing.B) { benchmarkUnixSo...
function BenchmarkUnixSocketProtocolV2Pub1k (line 1613) | func BenchmarkUnixSocketProtocolV2Pub1k(b *testing.B) { benchmarkUnixSo...
function BenchmarkUnixSocketProtocolV2Pub2k (line 1614) | func BenchmarkUnixSocketProtocolV2Pub2k(b *testing.B) { benchmarkUnixSo...
function BenchmarkUnixSocketProtocolV2Pub4k (line 1615) | func BenchmarkUnixSocketProtocolV2Pub4k(b *testing.B) { benchmarkUnixSo...
function BenchmarkUnixSocketProtocolV2Pub8k (line 1616) | func BenchmarkUnixSocketProtocolV2Pub8k(b *testing.B) { benchmarkUnixSo...
function BenchmarkUnixSocketProtocolV2Pub16k (line 1617) | func BenchmarkUnixSocketProtocolV2Pub16k(b *testing.B) { benchmarkUnixSo...
function BenchmarkUnixSocketProtocolV2Pub32k (line 1618) | func BenchmarkUnixSocketProtocolV2Pub32k(b *testing.B) { benchmarkUnixSo...
function BenchmarkUnixSocketProtocolV2Pub64k (line 1619) | func BenchmarkUnixSocketProtocolV2Pub64k(b *testing.B) { benchmarkUnixSo...
function BenchmarkUnixSocketProtocolV2Pub128k (line 1620) | func BenchmarkUnixSocketProtocolV2Pub128k(b *testing.B) {
function BenchmarkUnixSocketProtocolV2Pub256k (line 1623) | func BenchmarkUnixSocketProtocolV2Pub256k(b *testing.B) {
function BenchmarkUnixSocketProtocolV2Pub512k (line 1626) | func BenchmarkUnixSocketProtocolV2Pub512k(b *testing.B) {
function BenchmarkUnixSocketProtocolV2Pub1m (line 1629) | func BenchmarkUnixSocketProtocolV2Pub1m(b *testing.B) { benchmarkUnixSoc...
function benchmarkUnixSocketProtocolV2Sub (line 1631) | func benchmarkUnixSocketProtocolV2Sub(b *testing.B, size int) {
function subUnixSocketWorker (line 1668) | func subUnixSocketWorker(n int, workers int, addr net.Addr, topicName st...
function BenchmarkUnixSocketProtocolV2Sub256 (line 1710) | func BenchmarkUnixSocketProtocolV2Sub256(b *testing.B) { benchmarkUnixSo...
function BenchmarkUnixSocketProtocolV2Sub512 (line 1711) | func BenchmarkUnixSocketProtocolV2Sub512(b *testing.B) { benchmarkUnixSo...
function BenchmarkUnixSocketProtocolV2Sub1k (line 1712) | func BenchmarkUnixSocketProtocolV2Sub1k(b *testing.B) { benchmarkUnixSo...
function BenchmarkUnixSocketProtocolV2Sub2k (line 1713) | func BenchmarkUnixSocketProtocolV2Sub2k(b *testing.B) { benchmarkUnixSo...
function BenchmarkUnixSocketProtocolV2Sub4k (line 1714) | func BenchmarkUnixSocketProtocolV2Sub4k(b *testing.B) { benchmarkUnixSo...
function BenchmarkUnixSocketProtocolV2Sub8k (line 1715) | func BenchmarkUnixSocketProtocolV2Sub8k(b *testing.B) { benchmarkUnixSo...
function BenchmarkUnixSocketProtocolV2Sub16k (line 1716) | func BenchmarkUnixSocketProtocolV2Sub16k(b *testing.B) { benchmarkUnixSo...
function BenchmarkUnixSocketProtocolV2Sub32k (line 1717) | func BenchmarkUnixSocketProtocolV2Sub32k(b *testing.B) { benchmarkUnixSo...
function BenchmarkUnixSocketProtocolV2Sub64k (line 1718) | func BenchmarkUnixSocketProtocolV2Sub64k(b *testing.B) { benchmarkUnixSo...
function BenchmarkUnixSocketProtocolV2Sub128k (line 1719) | func BenchmarkUnixSocketProtocolV2Sub128k(b *testing.B) {
function BenchmarkUnixSocketProtocolV2Sub256k (line 1722) | func BenchmarkUnixSocketProtocolV2Sub256k(b *testing.B) {
function BenchmarkUnixSocketProtocolV2Sub512k (line 1725) | func BenchmarkUnixSocketProtocolV2Sub512k(b *testing.B) {
function BenchmarkUnixSocketProtocolV2Sub1m (line 1728) | func BenchmarkUnixSocketProtocolV2Sub1m(b *testing.B) { benchmarkUnixSoc...
function benchmarkUnixSocketProtocolV2MultiSub (line 1730) | func benchmarkUnixSocketProtocolV2MultiSub(b *testing.B, num int) {
function BenchmarkUnixSocketProtocolV2MultiSub2 (line 1772) | func BenchmarkUnixSocketProtocolV2MultiSub2(b *testing.B) {
function BenchmarkUnixSocketProtocolV2MultiSub1 (line 1775) | func BenchmarkUnixSocketProtocolV2MultiSub1(b *testing.B) {
function BenchmarkUnixSocketProtocolV2MultiSub4 (line 1778) | func BenchmarkUnixSocketProtocolV2MultiSub4(b *testing.B) {
function BenchmarkUnixSocketProtocolV2MultiSub8 (line 1781) | func BenchmarkUnixSocketProtocolV2MultiSub8(b *testing.B) {
function BenchmarkUnixSocketProtocolV2MultiSub16 (line 1784) | func BenchmarkUnixSocketProtocolV2MultiSub16(b *testing.B) {
FILE: nsqd/stats.go
type Stats (line 11) | type Stats struct
type ClientStats (line 16) | type ClientStats interface
type TopicStats (line 20) | type TopicStats struct
function NewTopicStats (line 32) | func NewTopicStats(t *Topic, channels []ChannelStats) TopicStats {
type ChannelStats (line 46) | type ChannelStats struct
function NewChannelStats (line 65) | func NewChannelStats(c *Channel, clients []ClientStats, clientCount int)...
type Topics (line 93) | type Topics
method Len (line 95) | func (t Topics) Len() int { return len(t) }
method Swap (line 96) | func (t Topics) Swap(i, j int) { t[i], t[j] = t[j], t[i] }
type TopicsByName (line 98) | type TopicsByName struct
method Less (line 102) | func (t TopicsByName) Less(i, j int) bool { return t.Topics[i].name < ...
type Channels (line 104) | type Channels
method Len (line 106) | func (c Channels) Len() int { return len(c) }
method Swap (line 107) | func (c Channels) Swap(i, j int) { c[i], c[j] = c[j], c[i] }
type ChannelsByName (line 109) | type ChannelsByName struct
method Less (line 113) | func (c ChannelsByName) Less(i, j int) bool { return c.Channels[i].nam...
method GetStats (line 115) | func (n *NSQD) GetStats(topic string, channel string, includeClients boo...
type memStats (line 186) | type memStats struct
function getMemStats (line 198) | func getMemStats() memStats {
FILE: nsqd/stats_test.go
function TestStats (line 17) | func TestStats(t *testing.T) {
function TestClientAttributes (line 68) | func TestClientAttributes(t *testing.T) {
function TestStatsChannelLocking (line 122) | func TestStatsChannelLocking(t *testing.T) {
FILE: nsqd/statsd.go
type Uint64Slice (line 14) | type Uint64Slice
method Len (line 16) | func (s Uint64Slice) Len() int {
method Swap (line 20) | func (s Uint64Slice) Swap(i, j int) {
method Less (line 24) | func (s Uint64Slice) Less(i, j int) bool {
method statsdLoop (line 28) | func (n *NSQD) statsdLoop() {
function percentile (line 163) | func percentile(perc float64, arr []uint64, length int) uint64 {
FILE: nsqd/tcp.go
constant typeConsumer (line 12) | typeConsumer = iota
constant typeProducer (line 13) | typeProducer
type Client (line 16) | type Client interface
type tcpServer (line 21) | type tcpServer struct
method Handle (line 26) | func (p *tcpServer) Handle(conn net.Conn) {
method Close (line 68) | func (p *tcpServer) Close() {
FILE: nsqd/topic.go
type Topic (line 16) | type Topic struct
method Start (line 86) | func (t *Topic) Start() {
method Exiting (line 94) | func (t *Topic) Exiting() bool {
method GetChannel (line 101) | func (t *Topic) GetChannel(channelName string) *Channel {
method getOrCreateChannel (line 118) | func (t *Topic) getOrCreateChannel(channelName string) (*Channel, bool) {
method GetExistingChannel (line 132) | func (t *Topic) GetExistingChannel(channelName string) (*Channel, erro...
method DeleteExistingChannel (line 143) | func (t *Topic) DeleteExistingChannel(channelName string) error {
method PutMessage (line 180) | func (t *Topic) PutMessage(m *Message) error {
method PutMessages (line 196) | func (t *Topic) PutMessages(msgs []*Message) error {
method put (line 220) | func (t *Topic) put(m *Message) error {
method Depth (line 243) | func (t *Topic) Depth() int64 {
method messagePump (line 249) | func (t *Topic) messagePump() {
method Delete (line 347) | func (t *Topic) Delete() error {
method Close (line 352) | func (t *Topic) Close() error {
method exit (line 356) | func (t *Topic) exit(deleted bool) error {
method Empty (line 405) | func (t *Topic) Empty() error {
method flush (line 418) | func (t *Topic) flush() error {
method AggregateChannelE2eProcessingLatency (line 442) | func (t *Topic) AggregateChannelE2eProcessingLatency() *quantile.Quant...
method Pause (line 464) | func (t *Topic) Pause() error {
method UnPause (line 468) | func (t *Topic) UnPause() error {
method doPause (line 472) | func (t *Topic) doPause(pause bool) error {
method IsPaused (line 487) | func (t *Topic) IsPaused() bool {
method GenerateID (line 491) | func (t *Topic) GenerateID() MessageID {
function NewTopic (line 45) | func NewTopic(topicName string, nsqd *NSQD, deleteCallback func(*Topic))...
FILE: nsqd/topic_test.go
function TestGetTopic (line 17) | func TestGetTopic(t *testing.T) {
function TestGetChannel (line 36) | func TestGetChannel(t *testing.T) {
type errorBackendQueue (line 55) | type errorBackendQueue struct
method Put (line 57) | func (d *errorBackendQueue) Put([]byte) error { return errors.N...
method ReadChan (line 58) | func (d *errorBackendQueue) ReadChan() <-chan []byte { return nil }
method Close (line 59) | func (d *errorBackendQueue) Close() error { return nil }
method Delete (line 60) | func (d *errorBackendQueue) Delete() error { return nil }
method Depth (line 61) | func (d *errorBackendQueue) Depth() int64 { return 0 }
method Empty (line 62) | func (d *errorBackendQueue) Empty() error { return nil }
type errorRecoveredBackendQueue (line 64) | type errorRecoveredBackendQueue struct
method Put (line 66) | func (d *errorRecoveredBackendQueue) Put([]byte) error { return nil }
function TestHealth (line 68) | func TestHealth(t *testing.T) {
function TestDeletes (line 117) | func TestDeletes(t *testing.T) {
function TestDeleteLast (line 142) | func TestDeleteLast(t *testing.T) {
function TestPause (line 165) | func TestPause(t *testing.T) {
function BenchmarkTopicPut (line 198) | func BenchmarkTopicPut(b *testing.B) {
function BenchmarkTopicToChannelPut (line 216) | func BenchmarkTopicToChannelPut(b *testing.B) {
FILE: nsqlookupd/client_v1.go
type ClientV1 (line 7) | type ClientV1 struct
method String (line 18) | func (c *ClientV1) String() string {
function NewClientV1 (line 12) | func NewClientV1(conn net.Conn) *ClientV1 {
FILE: nsqlookupd/http.go
type httpServer (line 15) | type httpServer struct
method ServeHTTP (line 64) | func (s *httpServer) ServeHTTP(w http.ResponseWriter, req *http.Reques...
method pingHandler (line 68) | func (s *httpServer) pingHandler(w http.ResponseWriter, req *http.Requ...
method doInfo (line 72) | func (s *httpServer) doInfo(w http.ResponseWriter, req *http.Request, ...
method doTopics (line 80) | func (s *httpServer) doTopics(w http.ResponseWriter, req *http.Request...
method doChannels (line 87) | func (s *httpServer) doChannels(w http.ResponseWriter, req *http.Reque...
method doLookup (line 104) | func (s *httpServer) doLookup(w http.ResponseWriter, req *http.Request...
method doCreateTopic (line 130) | func (s *httpServer) doCreateTopic(w http.ResponseWriter, req *http.Re...
method doDeleteTopic (line 152) | func (s *httpServer) doDeleteTopic(w http.ResponseWriter, req *http.Re...
method doTombstoneTopicProducer (line 178) | func (s *httpServer) doTombstoneTopicProducer(w http.ResponseWriter, r...
method doCreateChannel (line 206) | func (s *httpServer) doCreateChannel(w http.ResponseWriter, req *http....
method doDeleteChannel (line 228) | func (s *httpServer) doDeleteChannel(w http.ResponseWriter, req *http....
method doNodes (line 265) | func (s *httpServer) doNodes(w http.ResponseWriter, req *http.Request,...
method doDebug (line 310) | func (s *httpServer) doDebug(w http.ResponseWriter, req *http.Request,...
function newHTTPServer (line 20) | func newHTTPServer(l *NSQLookupd) *httpServer {
type node (line 252) | type node struct
FILE: nsqlookupd/http_test.go
type InfoDoc (line 18) | type InfoDoc struct
type ChannelsDoc (line 22) | type ChannelsDoc struct
type ErrMessage (line 26) | type ErrMessage struct
function bootstrapNSQCluster (line 30) | func bootstrapNSQCluster(t *testing.T) (string, []*nsqd.NSQD, *NSQLookup...
function makeTopic (line 78) | func makeTopic(nsqlookupd *NSQLookupd, topicName string) {
function makeChannel (line 83) | func makeChannel(nsqlookupd *NSQLookupd, topicName string, channelName s...
function TestPing (line 89) | func TestPing(t *testing.T) {
function TestInfo (line 107) | func TestInfo(t *testing.T) {
function TestCreateTopic (line 129) | func TestCreateTopic(t *testing.T) {
function TestDeleteTopic (line 182) | func TestDeleteTopic(t *testing.T) {
function TestGetChannels (line 237) | func TestGetChannels(t *testing.T) {
function TestCreateChannel (line 301) | func TestCreateChannel(t *testing.T) {
function TestDeleteChannel (line 386) | func TestDeleteChannel(t *testing.T) {
FILE: nsqlookupd/logger.go
type Logger (line 7) | type Logger
constant LOG_DEBUG (line 10) | LOG_DEBUG = lg.DEBUG
constant LOG_INFO (line 11) | LOG_INFO = lg.INFO
constant LOG_WARN (line 12) | LOG_WARN = lg.WARN
constant LOG_ERROR (line 13) | LOG_ERROR = lg.ERROR
constant LOG_FATAL (line 14) | LOG_FATAL = lg.FATAL
method logf (line 17) | func (n *NSQLookupd) logf(level lg.LogLevel, f string, args ...interface...
FILE: nsqlookupd/lookup_protocol_v1.go
type LookupProtocolV1 (line 20) | type LookupProtocolV1 struct
method NewClient (line 24) | func (p *LookupProtocolV1) NewClient(conn net.Conn) protocol.Client {
method IOLoop (line 28) | func (p *LookupProtocolV1) IOLoop(c protocol.Client) error {
method Exec (line 89) | func (p *LookupProtocolV1) Exec(client *ClientV1, reader *bufio.Reader...
method REGISTER (line 125) | func (p *LookupProtocolV1) REGISTER(client *ClientV1, reader *bufio.Re...
method UNREGISTER (line 151) | func (p *LookupProtocolV1) UNREGISTER(client *ClientV1, reader *bufio....
method IDENTIFY (line 200) | func (p *LookupProtocolV1) IDENTIFY(client *ClientV1, reader *bufio.Re...
method PING (line 263) | func (p *LookupProtocolV1) PING(client *ClientV1, params []string) ([]...
function getTopicChan (line 103) | func getTopicChan(command string, params []string) (string, string, erro...
FILE: nsqlookupd/lookup_protocol_v1_test.go
function TestIOLoopReturnsClientErrWhenSendFails (line 12) | func TestIOLoopReturnsClientErrWhenSendFails(t *testing.T) {
function TestIOLoopReturnsClientErrWhenSendSucceeds (line 21) | func TestIOLoopReturnsClientErrWhenSendSucceeds(t *testing.T) {
function testIOLoopReturnsClientErr (line 30) | func testIOLoopReturnsClientErr(t *testing.T, fakeConn test.FakeNetConn) {
FILE: nsqlookupd/nsqlookupd.go
type NSQLookupd (line 16) | type NSQLookupd struct
method Main (line 54) | func (l *NSQLookupd) Main() error {
method RealTCPAddr (line 78) | func (l *NSQLookupd) RealTCPAddr() *net.TCPAddr {
method RealHTTPAddr (line 82) | func (l *NSQLookupd) RealHTTPAddr() *net.TCPAddr {
method Exit (line 86) | func (l *NSQLookupd) Exit() {
function New (line 26) | func New(opts *Options) (*NSQLookupd, error) {
FILE: nsqlookupd/nsqlookupd_test.go
constant ConnectTimeout (line 16) | ConnectTimeout = 2 * time.Second
constant RequestTimeout (line 17) | RequestTimeout = 5 * time.Second
constant TCPPort (line 18) | TCPPort = 5000
constant HTTPPort (line 19) | HTTPPort = 5555
constant HostAddr (line 20) | HostAddr = "ip.address"
constant NSQDVersion (line 21) | NSQDVersion = "fake-version"
type ProducersDoc (line 24) | type ProducersDoc struct
type TopicsDoc (line 28) | type TopicsDoc struct
type LookupDoc (line 32) | type LookupDoc struct
function mustStartLookupd (line 37) | func mustStartLookupd(opts *Options) (*net.TCPAddr, *net.TCPAddr, *NSQLo...
function mustConnectLookupd (line 53) | func mustConnectLookupd(t *testing.T, tcpAddr *net.TCPAddr) net.Conn {
function identify (line 62) | func identify(t *testing.T, conn net.Conn) {
function TestBasicLookupd (line 76) | func TestBasicLookupd(t *testing.T) {
function TestChannelUnregister (line 150) | func TestChannelUnregister(t *testing.T) {
function TestTombstoneRecover (line 198) | func TestTombstoneRecover(t *testing.T) {
function TestTombstoneUnregister (line 246) | func TestTombstoneUnregister(t *testing.T) {
function TestInactiveNodes (line 288) | func TestInactiveNodes(t *testing.T) {
function TestTombstonedNodes (line 322) | func TestTombstonedNodes(t *testing.T) {
FILE: nsqlookupd/options.go
type Options (line 11) | type Options struct
function NewOptions (line 24) | func NewOptions() *Options {
FILE: nsqlookupd/registration_db.go
type RegistrationDB (line 10) | type RegistrationDB struct
method AddRegistration (line 64) | func (r *RegistrationDB) AddRegistration(k Registration) {
method AddProducer (line 74) | func (r *RegistrationDB) AddProducer(k Registration, p *Producer) bool {
method RemoveProducer (line 90) | func (r *RegistrationDB) RemoveProducer(k Registration, id string) (bo...
method RemoveRegistration (line 108) | func (r *RegistrationDB) RemoveRegistration(k Registration) {
method needFilter (line 114) | func (r *RegistrationDB) needFilter(key string, subkey string) bool {
method FindRegistrations (line 118) | func (r *RegistrationDB) FindRegistrations(category string, key string...
method FindProducers (line 138) | func (r *RegistrationDB) FindProducers(category string, key string, su...
method LookupRegistrations (line 163) | func (r *RegistrationDB) LookupRegistrations(id string) Registrations {
type Registration (line 15) | type Registration struct
method IsMatch (line 175) | func (k Registration) IsMatch(category string, key string, subkey stri...
type Registrations (line 20) | type Registrations
method Filter (line 188) | func (rr Registrations) Filter(category string, key string, subkey str...
method Keys (line 198) | func (rr Registrations) Keys() []string {
method SubKeys (line 206) | func (rr Registrations) SubKeys() []string {
type PeerInfo (line 22) | type PeerInfo struct
type Producer (line 35) | type Producer struct
method String (line 44) | func (p *Producer) String() string {
method Tombstone (line 48) | func (p *Producer) Tombstone() {
method IsTombstoned (line 53) | func (p *Producer) IsTombstoned(lifetime time.Duration) bool {
type Producers (line 41) | type Producers
method FilterByActive (line 214) | func (pp Producers) FilterByActive(inactivityTimeout time.Duration, to...
method PeerInfo (line 227) | func (pp Producers) PeerInfo() []*PeerInfo {
type ProducerMap (line 42) | type ProducerMap
function NewRegistrationDB (line 57) | func NewRegistrationDB() *RegistrationDB {
function ProducerMap2Slice (line 235) | func ProducerMap2Slice(pm ProducerMap) Producers {
FILE: nsqlookupd/registration_db_test.go
function TestRegistrationDB (line 12) | func TestRegistrationDB(t *testing.T) {
function fillRegDB (line 102) | func fillRegDB(registrations int, producers int) *RegistrationDB {
function benchmarkLookupRegistrations (line 122) | func benchmarkLookupRegistrations(b *testing.B, registrations int, produ...
function benchmarkRegister (line 131) | func benchmarkRegister(b *testing.B, registrations int, producers int) {
function benchmarkDoLookup (line 137) | func benchmarkDoLookup(b *testing.B, registrations int, producers int) {
function BenchmarkLookupRegistrations8x8 (line 148) | func BenchmarkLookupRegistrations8x8(b *testing.B) {
function BenchmarkLookupRegistrations8x64 (line 152) | func BenchmarkLookupRegistrations8x64(b *testing.B) {
function BenchmarkLookupRegistrations64x64 (line 156) | func BenchmarkLookupRegistrations64x64(b *testing.B) {
function BenchmarkLookupRegistrations64x512 (line 160) | func BenchmarkLookupRegistrations64x512(b *testing.B) {
function BenchmarkLookupRegistrations512x512 (line 164) | func BenchmarkLookupRegistrations512x512(b *testing.B) {
function BenchmarkLookupRegistrations512x2048 (line 168) | func BenchmarkLookupRegistrations512x2048(b *testing.B) {
function BenchmarkRegister8x8 (line 172) | func BenchmarkRegister8x8(b *testing.B) {
function BenchmarkRegister8x64 (line 176) | func BenchmarkRegister8x64(b *testing.B) {
function BenchmarkRegister64x64 (line 180) | func BenchmarkRegister64x64(b *testing.B) {
function BenchmarkRegister64x512 (line 184) | func BenchmarkRegister64x512(b *testing.B) {
function BenchmarkRegister512x512 (line 188) | func BenchmarkRegister512x512(b *testing.B) {
function BenchmarkRegister512x2048 (line 192) | func BenchmarkRegister512x2048(b *testing.B) {
function BenchmarkDoLookup8x8 (line 196) | func BenchmarkDoLookup8x8(b *testing.B) {
function BenchmarkDoLookup8x64 (line 200) | func BenchmarkDoLookup8x64(b *testing.B) {
function BenchmarkDoLookup64x64 (line 204) | func BenchmarkDoLookup64x64(b *testing.B) {
function BenchmarkDoLookup64x512 (line 208) | func BenchmarkDoLookup64x512(b *testing.B) {
function BenchmarkDoLookup512x512 (line 212) | func BenchmarkDoLookup512x512(b *testing.B) {
function BenchmarkDoLookup512x2048 (line 216) | func BenchmarkDoLookup512x2048(b *testing.B) {
FILE: nsqlookupd/tcp.go
type tcpServer (line 11) | type tcpServer struct
method Handle (line 16) | func (p *tcpServer) Handle(conn net.Conn) {
method Close (line 58) | func (p *tcpServer) Close() {
Condensed preview — 211 files, each showing path, character count, and a content snippet. Download the .json file or copy for the full structured content (1,300K chars).
[
{
"path": ".github/workflows/test.yml",
"chars": 1081,
"preview": "name: tests\n\non:\n push: {branches: [master]}\n pull_request: {branches: [master]}\n\njobs:\n test:\n runs-on: u"
},
{
"path": ".gitignore",
"chars": 730,
"preview": "/build/\ndist/\n.cover/\nprofile/\n\n# nsqd data from testing\n*.dat\n\n# nsqadmin\nnode_modules\n\n# apps\napps/nsqlookupd/nsqlooku"
},
{
"path": "AUTHORS",
"chars": 411,
"preview": "# For a complete listing, see https://github.com/nsqio/nsq/graphs/contributors\n\n# Original Authors\n\nMatt Reiferson <mrei"
},
{
"path": "CODE_OF_CONDUCT.md",
"chars": 1997,
"preview": "# Contributor Code of Conduct\n\nAs contributors and maintainers of this project, and in the interest of fostering an open"
},
{
"path": "CONTRIBUTING.md",
"chars": 1463,
"preview": "# Contributing\n\nThanks for your interest in contributing to NSQ!\n\n## Code of Conduct\n\nHelp us keep NSQ open and inclusiv"
},
{
"path": "ChangeLog.md",
"chars": 55856,
"preview": "# NSQ Changelog\n\n## Releases\n\n### 1.3.0 - 2023-12-26\n\n**Upgrading**\n\n * #1427 / #1373 / #1371 - fix staticcheck warnings"
},
{
"path": "Dockerfile",
"chars": 707,
"preview": "FROM golang:alpine AS build\n\nRUN apk update && apk add make gcc musl-dev\n\nRUN mkdir -p /go/src/github.com/nsqio/nsq\nCOPY"
},
{
"path": "LICENSE",
"chars": 1023,
"preview": "Permission is hereby granted, free of charge, to any person obtaining a copy\nof this software and associated documentati"
},
{
"path": "Makefile",
"chars": 1335,
"preview": "PREFIX=/usr/local\nBINDIR=${PREFIX}/bin\nDESTDIR=\nBLDDIR=build\nBLDFLAGS=\nEXT=\nifeq (${GOOS},windows)\n EXT=.exe\nendif\n\nA"
},
{
"path": "README.md",
"chars": 7617,
"preview": "<p align=\"center\">\n<img align=\"left\" width=\"175\" src=\"https://nsq.io/static/img/nsq_blue.png\">\n<ul>\n<li><strong>Source</"
},
{
"path": "apps/nsq_stat/nsq_stat.go",
"chars": 5034,
"preview": "// This is a utility application that polls /stats for all the producers\n// of the specified topic/channel and displays "
},
{
"path": "apps/nsq_tail/nsq_tail.go",
"chars": 3589,
"preview": "package main\n\nimport (\n\t\"flag\"\n\t\"fmt\"\n\t\"log\"\n\t\"math/rand\"\n\t\"os\"\n\t\"os/signal\"\n\t\"syscall\"\n\t\"time\"\n\n\t\"github.com/nsqio/go-n"
},
{
"path": "apps/nsq_to_file/file_logger.go",
"chars": 10638,
"preview": "package main\n\nimport (\n\t\"compress/gzip\"\n\t\"errors\"\n\t\"fmt\"\n\t\"io\"\n\t\"os\"\n\t\"path\"\n\t\"path/filepath\"\n\t\"strings\"\n\t\"time\"\n\n\t\"gith"
},
{
"path": "apps/nsq_to_file/nsq_to_file.go",
"chars": 4913,
"preview": "// This is a client that writes out to a file, and optionally rolls the file\n\npackage main\n\nimport (\n\t\"flag\"\n\t\"fmt\"\n\t\"lo"
},
{
"path": "apps/nsq_to_file/options.go",
"chars": 1927,
"preview": "package main\n\nimport \"time\"\n\ntype Options struct {\n\tTopics []string `flag:\"topic\"`\n\tTopicPattern "
},
{
"path": "apps/nsq_to_file/strftime.go",
"chars": 1801,
"preview": "// COPIED FROM https://github.com/jehiah/go-strftime\npackage main\n\nimport (\n\t\"time\"\n)\n\n// taken from time/format.go\nvar "
},
{
"path": "apps/nsq_to_file/topic_discoverer.go",
"chars": 2318,
"preview": "package main\n\nimport (\n\t\"os\"\n\t\"regexp\"\n\t\"sync\"\n\t\"time\"\n\n\t\"github.com/nsqio/go-nsq\"\n\t\"github.com/nsqio/nsq/internal/clust"
},
{
"path": "apps/nsq_to_http/http.go",
"chars": 871,
"preview": "package main\n\nimport (\n\t\"bytes\"\n\t\"fmt\"\n\t\"net/http\"\n\n\t\"github.com/nsqio/nsq/internal/version\"\n)\n\nvar httpclient *http.Cli"
},
{
"path": "apps/nsq_to_http/nsq_to_http.go",
"chars": 8460,
"preview": "// This is an NSQ client that reads the specified topic/channel\n// and performs HTTP requests (GET/POST) to the specifie"
},
{
"path": "apps/nsq_to_http/nsq_to_http_test.go",
"chars": 1526,
"preview": "// This is an NSQ client that reads the specified topic/channel\n// and performs HTTP requests (GET/POST) to the specifie"
},
{
"path": "apps/nsq_to_nsq/nsq_to_nsq.go",
"chars": 10797,
"preview": "// This is an NSQ client that reads the specified topic/channel\n// and re-publishes the messages to destination nsqd via"
},
{
"path": "apps/nsqadmin/main.go",
"chars": 4789,
"preview": "package main\n\nimport (\n\t\"flag\"\n\t\"fmt\"\n\t\"os\"\n\t\"path/filepath\"\n\t\"sync\"\n\t\"syscall\"\n\n\t\"github.com/BurntSushi/toml\"\n\t\"github."
},
{
"path": "apps/nsqadmin/main_test.go",
"chars": 551,
"preview": "package main\n\nimport (\n\t\"testing\"\n\n\t\"github.com/mreiferson/go-options\"\n\t\"github.com/nsqio/nsq/internal/lg\"\n\t\"github.com/"
},
{
"path": "apps/nsqadmin/options.go",
"chars": 408,
"preview": "package main\n\nimport (\n\t\"fmt\"\n\n\t\"github.com/nsqio/nsq/internal/lg\"\n)\n\ntype config map[string]interface{}\n\n// Validate se"
},
{
"path": "apps/nsqd/README.md",
"chars": 147,
"preview": "## nsqd\r\n\r\n`nsqd` is the daemon that receives, queues, and delivers messages to clients.\r\n\r\nRead the [docs](https://nsq."
},
{
"path": "apps/nsqd/main.go",
"chars": 2471,
"preview": "package main\n\nimport (\n\t\"context\"\n\t\"flag\"\n\t\"fmt\"\n\t\"math/rand\"\n\t\"os\"\n\t\"sync\"\n\t\"syscall\"\n\t\"time\"\n\n\t\"github.com/BurntSushi/"
},
{
"path": "apps/nsqd/main_test.go",
"chars": 883,
"preview": "package main\n\nimport (\n\t\"crypto/tls\"\n\t\"os\"\n\t\"testing\"\n\n\t\"github.com/BurntSushi/toml\"\n\t\"github.com/mreiferson/go-options\""
},
{
"path": "apps/nsqd/options.go",
"chars": 10097,
"preview": "package main\n\nimport (\n\t\"crypto/tls\"\n\t\"flag\"\n\t\"fmt\"\n\t\"strconv\"\n\t\"strings\"\n\n\t\"github.com/nsqio/nsq/internal/app\"\n\t\"github"
},
{
"path": "apps/nsqlookupd/README.md",
"chars": 214,
"preview": "## nsqlookupd\r\n\r\n`nsqlookupd` is the daemon that manages topology metadata and serves client requests to\r\ndiscover the l"
},
{
"path": "apps/nsqlookupd/main.go",
"chars": 2785,
"preview": "package main\n\nimport (\n\t\"flag\"\n\t\"fmt\"\n\t\"os\"\n\t\"path/filepath\"\n\t\"sync\"\n\t\"syscall\"\n\n\t\"github.com/BurntSushi/toml\"\n\t\"github."
},
{
"path": "apps/nsqlookupd/main_test.go",
"chars": 557,
"preview": "package main\n\nimport (\n\t\"testing\"\n\n\t\"github.com/mreiferson/go-options\"\n\t\"github.com/nsqio/nsq/internal/lg\"\n\t\"github.com/"
},
{
"path": "apps/nsqlookupd/options.go",
"chars": 408,
"preview": "package main\n\nimport (\n\t\"fmt\"\n\n\t\"github.com/nsqio/nsq/internal/lg\"\n)\n\ntype config map[string]interface{}\n\n// Validate se"
},
{
"path": "apps/to_nsq/README.md",
"chars": 808,
"preview": "# to_nsq\n\nA tool for publishing to an nsq topic with data from `stdin`.\n\n## Usage\n\n```\nUsage of ./to_nsq:\n -delimiter s"
},
{
"path": "apps/to_nsq/to_nsq.go",
"chars": 3301,
"preview": "// This is an NSQ client that publishes incoming messages from\n// stdin to the specified topic.\n\npackage main\n\nimport (\n"
},
{
"path": "bench/bench.py",
"chars": 11977,
"preview": "#!/usr/bin/env python3\n\n#\n# This script bootstraps an NSQ cluster in EC2 and runs benchmarks.\n#\n# Requires python3 and t"
},
{
"path": "bench/bench_channels/bench_channels.go",
"chars": 1782,
"preview": "package main\n\nimport (\n\t\"bufio\"\n\t\"flag\"\n\t\"fmt\"\n\t\"net\"\n\t\"sync\"\n\t\"time\"\n\n\t\"github.com/nsqio/go-nsq\"\n)\n\nvar (\n\tnum ="
},
{
"path": "bench/bench_reader/bench_reader.go",
"chars": 3002,
"preview": "package main\n\nimport (\n\t\"bufio\"\n\t\"errors\"\n\t\"flag\"\n\t\"fmt\"\n\t\"log\"\n\t\"net\"\n\t\"runtime\"\n\t\"sync\"\n\t\"sync/atomic\"\n\t\"time\"\n\n\t\"gith"
},
{
"path": "bench/bench_writer/bench_writer.go",
"chars": 2825,
"preview": "package main\n\nimport (\n\t\"bufio\"\n\t\"flag\"\n\t\"fmt\"\n\t\"log\"\n\t\"net\"\n\t\"runtime\"\n\t\"sync\"\n\t\"sync/atomic\"\n\t\"time\"\n\n\t\"github.com/nsq"
},
{
"path": "bench/requirements.txt",
"chars": 43,
"preview": "tornado==4.3\nparamiko==1.16.0\nboto==2.39.0\n"
},
{
"path": "bench.sh",
"chars": 1260,
"preview": "#!/bin/bash\nreadonly messageSize=\"${1:-200}\"\nreadonly batchSize=\"${2:-200}\"\nreadonly memQueueSize=\"${3:-1000000}\"\nreadon"
},
{
"path": "contrib/nsq.spec",
"chars": 1192,
"preview": "%define name nsq\n%define version 1.1.1-alpha\n%define release 1\n%define path usr/local\n%define group Database/Application"
},
{
"path": "contrib/nsqadmin.cfg.example",
"chars": 1911,
"preview": "## log verbosity level: debug, info, warn, error, or fatal\nlog_level = \"info\"\n\n## log message prefix (default \"[nsqadmin"
},
{
"path": "contrib/nsqd.cfg.example",
"chars": 3332,
"preview": "## log verbosity level: debug, info, warn, error, or fatal\nlog_level = \"info\"\n\n## unique identifier (int) for this worke"
},
{
"path": "contrib/nsqlookupd.cfg.example",
"chars": 561,
"preview": "## log verbosity level: debug, info, warn, error, or fatal\nlog_level = \"info\"\n\n## <addr>:<port> to listen on for TCP cli"
},
{
"path": "coverage.sh",
"chars": 1444,
"preview": "#!/bin/bash\n# Generate test coverage statistics for Go packages.\n#\n# Works around the fact that `go test -coverprofile` "
},
{
"path": "dist.sh",
"chars": 2006,
"preview": "#!/bin/bash\n\n# 1. commit to bump the version and update the changelog/readme\n# 2. tag that commit\n# 3. use dist.sh to pr"
},
{
"path": "fmt.sh",
"chars": 53,
"preview": "#!/bin/bash\nfind . -name \"*.go\" | xargs goimports -w\n"
},
{
"path": "go.mod",
"chars": 696,
"preview": "module github.com/nsqio/nsq\n\ngo 1.17\n\nrequire (\n\tgithub.com/BurntSushi/toml v1.3.2\n\tgithub.com/bitly/go-hostpool v0.1.0\n"
},
{
"path": "go.sum",
"chars": 4169,
"preview": "github.com/BurntSushi/toml v1.3.2 h1:o7IhLm0Msx3BaB+n3Ag7L8EVlByGnpq14C4YWiu/gL8=\ngithub.com/BurntSushi/toml v1.3.2/go.m"
},
{
"path": "internal/app/float_array.go",
"chars": 764,
"preview": "package app\n\nimport (\n\t\"fmt\"\n\t\"log\"\n\t\"sort\"\n\t\"strconv\"\n\t\"strings\"\n)\n\ntype FloatArray []float64\n\nfunc (a *FloatArray) Get"
},
{
"path": "internal/app/string_array.go",
"chars": 279,
"preview": "package app\n\nimport (\n\t\"strings\"\n)\n\ntype StringArray []string\n\nfunc (a *StringArray) Get() interface{} { return []string"
},
{
"path": "internal/auth/authorizations.go",
"chars": 3876,
"preview": "package auth\n\nimport (\n\t\"crypto/tls\"\n\t\"errors\"\n\t\"fmt\"\n\t\"math/rand\"\n\t\"net/url\"\n\t\"regexp\"\n\t\"strings\"\n\t\"time\"\n\n\t\"github.com"
},
{
"path": "internal/clusterinfo/data.go",
"chars": 23902,
"preview": "package clusterinfo\n\nimport (\n\t\"fmt\"\n\t\"net\"\n\t\"net/url\"\n\t\"sort\"\n\t\"strconv\"\n\t\"strings\"\n\t\"sync\"\n\n\t\"github.com/blang/semver\""
},
{
"path": "internal/clusterinfo/producer_test.go",
"chars": 1163,
"preview": "package clusterinfo\n\nimport \"testing\"\n\nfunc TestHostNameAddresses(t *testing.T) {\n\tp := &Producer{\n\t\tBroadcastAddress: \""
},
{
"path": "internal/clusterinfo/types.go",
"chars": 12399,
"preview": "package clusterinfo\n\nimport (\n\t\"encoding/json\"\n\t\"net\"\n\t\"sort\"\n\t\"strconv\"\n\t\"time\"\n\n\t\"github.com/blang/semver\"\n\t\"github.co"
},
{
"path": "internal/dirlock/dirlock.go",
"chars": 658,
"preview": "//go:build !windows && !illumos\n// +build !windows,!illumos\n\npackage dirlock\n\nimport (\n\t\"fmt\"\n\t\"os\"\n\t\"syscall\"\n)\n\ntype D"
},
{
"path": "internal/dirlock/dirlock_illumos.go",
"chars": 257,
"preview": "//go:build illumos\n// +build illumos\n\npackage dirlock\n\ntype DirLock struct {\n\tdir string\n}\n\nfunc New(dir string) *DirLoc"
},
{
"path": "internal/dirlock/dirlock_windows.go",
"chars": 257,
"preview": "//go:build windows\n// +build windows\n\npackage dirlock\n\ntype DirLock struct {\n\tdir string\n}\n\nfunc New(dir string) *DirLoc"
},
{
"path": "internal/http_api/api_request.go",
"chars": 3540,
"preview": "package http_api\n\nimport (\n\t\"bytes\"\n\t\"crypto/tls\"\n\t\"encoding/json\"\n\t\"fmt\"\n\t\"io\"\n\t\"net\"\n\t\"net/http\"\n\t\"net/url\"\n\t\"strconv\""
},
{
"path": "internal/http_api/api_response.go",
"chars": 3706,
"preview": "package http_api\n\nimport (\n\t\"encoding/json\"\n\t\"fmt\"\n\t\"io\"\n\t\"net/http\"\n\t\"time\"\n\n\t\"github.com/julienschmidt/httprouter\"\n\t\"g"
},
{
"path": "internal/http_api/compress.go",
"chars": 2096,
"preview": "// Copyright 2013 The Gorilla Authors. All rights reserved.\n// Use of this source code is governed by a BSD-style\n// lic"
},
{
"path": "internal/http_api/http_server.go",
"chars": 801,
"preview": "package http_api\n\nimport (\n\t\"errors\"\n\t\"fmt\"\n\t\"log\"\n\t\"net\"\n\t\"net/http\"\n\n\t\"github.com/nsqio/nsq/internal/lg\"\n)\n\ntype logWr"
},
{
"path": "internal/http_api/req_params.go",
"chars": 717,
"preview": "package http_api\n\nimport (\n\t\"errors\"\n\t\"io\"\n\t\"net/http\"\n\t\"net/url\"\n)\n\ntype ReqParams struct {\n\turl.Values\n\tBody []byte\n}\n"
},
{
"path": "internal/http_api/topic_channel_args.go",
"chars": 659,
"preview": "package http_api\n\nimport (\n\t\"errors\"\n\n\t\"github.com/nsqio/nsq/internal/protocol\"\n)\n\ntype getter interface {\n\tGet(key stri"
},
{
"path": "internal/lg/lg.go",
"chars": 1620,
"preview": "// Package lg provides leveled logging\npackage lg\n\nimport (\n\t\"fmt\"\n\t\"log\"\n\t\"os\"\n\t\"strings\"\n)\n\nconst (\n\tDEBUG = LogLevel("
},
{
"path": "internal/lg/lg_test.go",
"chars": 740,
"preview": "package lg\n\nimport (\n\t\"testing\"\n\n\t\"github.com/nsqio/nsq/internal/test\"\n)\n\ntype mockLogger struct {\n\tCount int\n}\n\nfunc (l"
},
{
"path": "internal/pqueue/pqueue.go",
"chars": 1292,
"preview": "package pqueue\n\nimport (\n\t\"container/heap\"\n)\n\ntype Item struct {\n\tValue interface{}\n\tPriority int64\n\tIndex int\n}\n\n"
},
{
"path": "internal/pqueue/pqueue_test.go",
"chars": 1542,
"preview": "package pqueue\n\nimport (\n\t\"container/heap\"\n\t\"math/rand\"\n\t\"path/filepath\"\n\t\"reflect\"\n\t\"runtime\"\n\t\"sort\"\n\t\"testing\"\n)\n\nfun"
},
{
"path": "internal/protocol/byte_base10.go",
"chars": 393,
"preview": "package protocol\n\nimport (\n\t\"errors\"\n)\n\nvar errBase10 = errors.New(\"failed to convert to Base10\")\n\nfunc ByteToBase10(b ["
},
{
"path": "internal/protocol/byte_base10_test.go",
"chars": 437,
"preview": "package protocol\n\nimport (\n\t\"testing\"\n)\n\nvar result uint64\n\nfunc BenchmarkByteToBase10Valid(b *testing.B) {\n\tbt := []byt"
},
{
"path": "internal/protocol/errors.go",
"chars": 1295,
"preview": "package protocol\n\ntype ChildErr interface {\n\tParent() error\n}\n\n// ClientErr provides a way for NSQ daemons to log a huma"
},
{
"path": "internal/protocol/names.go",
"chars": 535,
"preview": "package protocol\n\nimport (\n\t\"regexp\"\n)\n\nvar validTopicChannelNameRegex = regexp.MustCompile(`^[.a-zA-Z0-9_-]+(#ephemeral"
},
{
"path": "internal/protocol/protocol.go",
"chars": 1172,
"preview": "package protocol\n\nimport (\n\t\"encoding/binary\"\n\t\"io\"\n\t\"net\"\n)\n\ntype Client interface {\n\tClose() error\n}\n\n// Protocol desc"
},
{
"path": "internal/protocol/tcp_server.go",
"chars": 1086,
"preview": "package protocol\n\nimport (\n\t\"errors\"\n\t\"fmt\"\n\t\"net\"\n\t\"runtime\"\n\t\"sync\"\n\n\t\"github.com/nsqio/nsq/internal/lg\"\n)\n\ntype TCPHa"
},
{
"path": "internal/quantile/aggregate.go",
"chars": 2260,
"preview": "package quantile\n\nimport (\n\t\"encoding/json\"\n\t\"math\"\n\t\"sort\"\n)\n\ntype E2eProcessingLatencyAggregate struct {\n\tCount "
},
{
"path": "internal/quantile/quantile.go",
"chars": 2562,
"preview": "package quantile\n\nimport (\n\t\"strings\"\n\t\"sync\"\n\t\"time\"\n\n\t\"github.com/bmizerany/perks/quantile\"\n\t\"github.com/nsqio/nsq/int"
},
{
"path": "internal/statsd/client.go",
"chars": 777,
"preview": "package statsd\n\nimport (\n\t\"fmt\"\n\t\"io\"\n)\n\ntype Client struct {\n\tw io.Writer\n\tprefix string\n}\n\nfunc NewClient(w io.Wr"
},
{
"path": "internal/statsd/host.go",
"chars": 145,
"preview": "package statsd\n\nimport (\n\t\"strings\"\n)\n\nfunc HostKey(h string) string {\n\treturn strings.Replace(strings.Replace(h, \".\", \""
},
{
"path": "internal/stringy/slice.go",
"chars": 595,
"preview": "package stringy\n\nfunc Add(s []string, a string) []string {\n\tfor _, existing := range s {\n\t\tif a == existing {\n\t\t\treturn "
},
{
"path": "internal/stringy/slice_test.go",
"chars": 620,
"preview": "package stringy_test\n\nimport (\n\t\"testing\"\n\n\t\"github.com/nsqio/nsq/internal/stringy\"\n)\n\nfunc BenchmarkUniq(b *testing.B) "
},
{
"path": "internal/stringy/template.go",
"chars": 330,
"preview": "package stringy\n\nimport (\n\t\"fmt\"\n)\n\nfunc NanoSecondToHuman(v float64) string {\n\tvar suffix string\n\tswitch {\n\tcase v > 10"
},
{
"path": "internal/test/assertions.go",
"chars": 1363,
"preview": "package test\n\nimport (\n\t\"path/filepath\"\n\t\"reflect\"\n\t\"runtime\"\n\t\"testing\"\n)\n\nfunc Equal(t *testing.T, expected, actual in"
},
{
"path": "internal/test/fakes.go",
"chars": 1828,
"preview": "package test\n\nimport (\n\t\"net\"\n\t\"time\"\n)\n\ntype FakeNetConn struct {\n\tReadFunc func([]byte) (int, error)\n\tWrit"
},
{
"path": "internal/test/logger.go",
"chars": 314,
"preview": "package test\n\ntype Logger interface {\n\tOutput(maxdepth int, s string) error\n}\n\ntype tbLog interface {\n\tLog(...interface{"
},
{
"path": "internal/util/rand.go",
"chars": 394,
"preview": "package util\n\nimport (\n\t\"math/rand\"\n)\n\nfunc UniqRands(quantity int, maxval int) []int {\n\tif maxval < quantity {\n\t\tquanti"
},
{
"path": "internal/util/unix_socket.go",
"chars": 161,
"preview": "package util\n\nimport (\n\t\"net\"\n)\n\nfunc TypeOfAddr(addr string) string {\n\tif _, _, err := net.SplitHostPort(addr); err == "
},
{
"path": "internal/util/util_test.go",
"chars": 855,
"preview": "package util\n\nimport (\n\t\"testing\"\n\n\t\"github.com/nsqio/nsq/internal/test\"\n)\n\nfunc BenchmarkUniqRands5of5(b *testing.B) {\n"
},
{
"path": "internal/util/wait_group_wrapper.go",
"chars": 177,
"preview": "package util\n\nimport (\n\t\"sync\"\n)\n\ntype WaitGroupWrapper struct {\n\tsync.WaitGroup\n}\n\nfunc (w *WaitGroupWrapper) Wrap(cb f"
},
{
"path": "internal/version/binary.go",
"chars": 181,
"preview": "package version\n\nimport (\n\t\"fmt\"\n\t\"runtime\"\n)\n\nconst Binary = \"1.3.0\"\n\nfunc String(app string) string {\n\treturn fmt.Spri"
},
{
"path": "internal/writers/boundary_buffered_writer.go",
"chars": 514,
"preview": "package writers\n\nimport (\n\t\"bufio\"\n\t\"io\"\n)\n\ntype BoundaryBufferedWriter struct {\n\tbw *bufio.Writer\n}\n\nfunc NewBoundaryBu"
},
{
"path": "internal/writers/spread_writer.go",
"chars": 917,
"preview": "package writers\n\nimport (\n\t\"io\"\n\t\"time\"\n)\n\ntype SpreadWriter struct {\n\tw io.Writer\n\tinterval time.Duration\n\tbuf "
},
{
"path": "nsqadmin/.eslintrc",
"chars": 2214,
"preview": "{\n \"env\": {\n \"browser\": true,\n \"es2020\": true\n },\n // Rule docs: http://eslint.org/docs/rules/\n \"rules\": {\n "
},
{
"path": "nsqadmin/README.md",
"chars": 595,
"preview": "## nsqadmin\n\n`nsqadmin` is a Web UI to view aggregated cluster stats in realtime and perform various\nadministrative task"
},
{
"path": "nsqadmin/gulp",
"chars": 45,
"preview": "#!/bin/sh\nexec ./node_modules/.bin/gulp \"$@\"\n"
},
{
"path": "nsqadmin/gulpfile.js",
"chars": 4670,
"preview": "var browserify = require('browserify');\nvar clean = require('gulp-clean');\nvar gulp = require('gulp');\nvar notify = requ"
},
{
"path": "nsqadmin/http.go",
"chars": 27038,
"preview": "package nsqadmin\n\nimport (\n\t\"encoding/json\"\n\t\"fmt\"\n\t\"html/template\"\n\t\"io\"\n\t\"mime\"\n\t\"net\"\n\t\"net/http\"\n\t\"net/http/httputil"
},
{
"path": "nsqadmin/http_test.go",
"chars": 18973,
"preview": "package nsqadmin\n\nimport (\n\t\"bytes\"\n\t\"encoding/json\"\n\t\"fmt\"\n\t\"io\"\n\t\"net\"\n\t\"net/http\"\n\t\"os\"\n\t\"strconv\"\n\t\"testing\"\n\t\"time\""
},
{
"path": "nsqadmin/logger.go",
"chars": 363,
"preview": "package nsqadmin\n\nimport (\n\t\"github.com/nsqio/nsq/internal/lg\"\n)\n\ntype Logger lg.Logger\n\nconst (\n\tLOG_DEBUG = lg.DEBUG\n\t"
},
{
"path": "nsqadmin/notify.go",
"chars": 1727,
"preview": "package nsqadmin\n\nimport (\n\t\"encoding/base64\"\n\t\"net/http\"\n\t\"net/url\"\n\t\"os\"\n\t\"strings\"\n\t\"time\"\n)\n\ntype AdminAction struct"
},
{
"path": "nsqadmin/nsqadmin.go",
"chars": 5224,
"preview": "package nsqadmin\n\nimport (\n\t\"bytes\"\n\t\"crypto/tls\"\n\t\"crypto/x509\"\n\t\"encoding/json\"\n\t\"errors\"\n\t\"fmt\"\n\t\"log\"\n\t\"net\"\n\t\"net/h"
},
{
"path": "nsqadmin/nsqadmin_test.go",
"chars": 2463,
"preview": "package nsqadmin\n\nimport (\n\t\"fmt\"\n\t\"net\"\n\t\"net/http\"\n\t\"net/url\"\n\t\"os\"\n\t\"testing\"\n\n\t\"github.com/nsqio/nsq/internal/lg\"\n\t\""
},
{
"path": "nsqadmin/options.go",
"chars": 2148,
"preview": "package nsqadmin\n\nimport (\n\t\"time\"\n\n\t\"github.com/nsqio/nsq/internal/lg\"\n)\n\ntype Options struct {\n\tLogLevel lg.LogLevel "
},
{
"path": "nsqadmin/package.json",
"chars": 897,
"preview": "{\n \"name\": \"nsqadmin\",\n \"version\": \"0.3.0\",\n \"description\": \"operational dashboard for NSQ (https://nsq.io/)\",\n \"rep"
},
{
"path": "nsqadmin/static/build/base.css",
"chars": 4230,
"preview": ".red {\n color: #c30;\n}\n\n.red:hover {\n color: #f30;\n}\n\n.bold {\n font-weight: bold;\n}\n\n.graph-row td {\n text-align: ce"
},
{
"path": "nsqadmin/static/build/index.html",
"chars": 1311,
"preview": "<!doctype html>\n<html>\n<head>\n <title>nsqadmin</title>\n <link rel=\"icon\" type=\"image/png\" href=\"{{basePath \"/stati"
},
{
"path": "nsqadmin/static/build/main.js",
"chars": 217092,
"preview": "!function l(a,o,r){function i(e,n){if(!o[e]){if(!a[e]){var t=\"function\"==typeof require&&require;if(!n&&t)return t(e,!0)"
},
{
"path": "nsqadmin/static/build/vendor.js",
"chars": 150015,
"preview": "require=function r(i,o,a){function s(t,e){if(!o[t]){if(!i[t]){var n=\"function\"==typeof require&&require;if(!e&&n)return "
},
{
"path": "nsqadmin/static/css/base.scss",
"chars": 4521,
"preview": ".red {\n color: #c30;\n}\n\n.red:hover {\n color: #f30;\n}\n\n.bold {\n font-weight: bold;\n}\n\n.graph-row td {\n text-a"
},
{
"path": "nsqadmin/static/html/index.html",
"chars": 1311,
"preview": "<!doctype html>\n<html>\n<head>\n <title>nsqadmin</title>\n <link rel=\"icon\" type=\"image/png\" href=\"{{basePath \"/stati"
},
{
"path": "nsqadmin/static/js/app_state.js",
"chars": 1588,
"preview": "var Backbone = require('backbone');\nvar _ = require('underscore');\n\nvar AppState = Backbone.Model.extend({\n defaults:"
},
{
"path": "nsqadmin/static/js/collections/nodes.js",
"chars": 883,
"preview": "var Backbone = require('backbone');\n\nvar AppState = require('../app_state');\n\nvar NodeModel = require('../models/node');"
},
{
"path": "nsqadmin/static/js/collections/topics.js",
"chars": 621,
"preview": "var _ = require('underscore');\nvar Backbone = require('backbone');\n\nvar AppState = require('../app_state');\n\nvar Topic ="
},
{
"path": "nsqadmin/static/js/lib/ajax_setup.js",
"chars": 398,
"preview": "var $ = require('jquery');\nvar _ = require('underscore');\n\n// Set up some headers and options for every request.\n$.ajaxP"
},
{
"path": "nsqadmin/static/js/lib/handlebars_helpers.js",
"chars": 9330,
"preview": "var $ = require('jquery');\nvar _ = require('underscore');\nvar Handlebars = require('hbsfy/runtime');\n\nvar AppState = req"
},
{
"path": "nsqadmin/static/js/lib/pubsub.js",
"chars": 227,
"preview": "var _ = require('underscore');\nvar Backbone = require('backbone');\n\nvar Pubsub = _.clone(Backbone.Events);\n\n// making th"
},
{
"path": "nsqadmin/static/js/main.js",
"chars": 515,
"preview": "var $ = require('jquery');\nvar Backbone = require('backbone');\n\n// var Pubsub = require('./lib/pubsub');\nvar Router = re"
},
{
"path": "nsqadmin/static/js/models/channel.js",
"chars": 2474,
"preview": "var _ = require('underscore');\n\nvar AppState = require('../app_state');\nvar Backbone = require('backbone');\n\nvar Channel"
},
{
"path": "nsqadmin/static/js/models/node.js",
"chars": 536,
"preview": "var AppState = require('../app_state');\nvar Backbone = require('backbone');\n\nvar NodeModel = Backbone.Model.extend({\n "
},
{
"path": "nsqadmin/static/js/models/topic.js",
"chars": 942,
"preview": "var _ = require('underscore');\n\nvar AppState = require('../app_state');\nvar Backbone = require('backbone');\n\nvar Topic ="
},
{
"path": "nsqadmin/static/js/router.js",
"chars": 1471,
"preview": "var Backbone = require('backbone');\n\nvar AppState = require('./app_state');\nvar Pubsub = require('./lib/pubsub');\n\n\nvar "
},
{
"path": "nsqadmin/static/js/views/app.js",
"chars": 5448,
"preview": "var $ = require('jquery');\n\nwindow.jQuery = $;\nvar bootstrap = require('bootstrap'); //eslint-disable-line no-unused-var"
},
{
"path": "nsqadmin/static/js/views/base.js",
"chars": 3552,
"preview": "var $ = require('jquery');\nvar _ = require('underscore');\nvar Backbone = require('backbone');\n\nvar AppState = require('."
},
{
"path": "nsqadmin/static/js/views/channel.hbs",
"chars": 14378,
"preview": "{{> warning}}\n{{> error}}\n\n<ol class=\"breadcrumb\">\n <li><a class=\"link\" href=\"{{basePath \"/\"}}\">Streams</a>\n <li><a cl"
},
{
"path": "nsqadmin/static/js/views/channel.js",
"chars": 2457,
"preview": "var $ = require('jquery');\n\nwindow.jQuery = $;\nvar bootstrap = require('bootstrap'); //eslint-disable-line no-unused-var"
},
{
"path": "nsqadmin/static/js/views/counter.hbs",
"chars": 3888,
"preview": "<style>\nbody {\n background-color:#000;\n}\n/*\n * .numbers : The container for .number\n */\n\n#numbers {\n text-align:ce"
},
{
"path": "nsqadmin/static/js/views/counter.js",
"chars": 4178,
"preview": "var _ = require('underscore');\nvar $ = require('jquery');\n\nvar AppState = require('../app_state');\n\nvar BaseView = requi"
},
{
"path": "nsqadmin/static/js/views/error.hbs",
"chars": 208,
"preview": "<div class=\"row\" id=\"error\" {{#unless message}}style=\"display: none;\"{{/unless}}>\n <div class=\"col-md-12\">\n <d"
},
{
"path": "nsqadmin/static/js/views/header.hbs",
"chars": 2082,
"preview": "<nav class=\"navbar navbar-inverse navbar-static-top\">\n <div class=\"container-fluid\">\n <div class=\"navbar-heade"
},
{
"path": "nsqadmin/static/js/views/header.js",
"chars": 1034,
"preview": "var _ = require('underscore');\nvar $ = require('jquery');\n\nvar AppState = require('../app_state');\n\nvar BaseView = requi"
},
{
"path": "nsqadmin/static/js/views/lookup.hbs",
"chars": 2462,
"preview": "{{> warning}}\n{{> error}}\n\n<div class=\"row\">\n <div class=\"col-md-12\">\n <h2>Lookup</h2>\n </div>\n</div>\n\n{{#u"
},
{
"path": "nsqadmin/static/js/views/lookup.js",
"chars": 2635,
"preview": "var _ = require('underscore');\nvar $ = require('jquery');\n\nvar AppState = require('../app_state');\nvar Pubsub = require("
},
{
"path": "nsqadmin/static/js/views/node.hbs",
"chars": 7292,
"preview": "{{> warning}}\n{{> error}}\n\n<ol class=\"breadcrumb\">\n <li><a class=\"link\" href=\"{{basePath \"/nodes\"}}\">Nodes</a>\n <l"
},
{
"path": "nsqadmin/static/js/views/node.js",
"chars": 746,
"preview": "var Pubsub = require('../lib/pubsub');\nvar AppState = require('../app_state');\n\nvar BaseView = require('./base');\n\nvar N"
},
{
"path": "nsqadmin/static/js/views/nodes.hbs",
"chars": 2113,
"preview": "{{> warning}}\n{{> error}}\n\n<div class=\"row\">\n <div class=\"col-md-12\">\n <h2>NSQd Nodes ({{collection.length}})<"
},
{
"path": "nsqadmin/static/js/views/nodes.js",
"chars": 1047,
"preview": "var $ = require('jquery');\n\nvar Pubsub = require('../lib/pubsub');\nvar AppState = require('../app_state');\n\nvar BaseView"
},
{
"path": "nsqadmin/static/js/views/spinner.hbs",
"chars": 136,
"preview": "<div class=\"bubblingG\">\n <span id=\"bubblingG_1\"></span>\n <span id=\"bubblingG_2\"></span>\n <span id=\"bubblingG_3\""
},
{
"path": "nsqadmin/static/js/views/topic.hbs",
"chars": 10694,
"preview": "{{> warning}}\n{{> error}}\n\n<ol class=\"breadcrumb\">\n <li><a class=\"link\" href=\"{{basePath \"/\"}}\">Streams</a>\n <li class"
},
{
"path": "nsqadmin/static/js/views/topic.js",
"chars": 1909,
"preview": "var $ = require('jquery');\n\nwindow.jQuery = $;\nvar bootstrap = require('bootstrap'); //eslint-disable-line no-unused-var"
},
{
"path": "nsqadmin/static/js/views/topics.hbs",
"chars": 1383,
"preview": "{{> warning}}\n{{> error}}\n\n<div class=\"row\">\n <div class=\"col-md-12\">\n <h2>Topics</h2>\n </div>\n</div>\n\n<div"
},
{
"path": "nsqadmin/static/js/views/topics.js",
"chars": 846,
"preview": "var Pubsub = require('../lib/pubsub');\nvar AppState = require('../app_state');\n\nvar BaseView = require('./base');\nvar To"
},
{
"path": "nsqadmin/static/js/views/warning.hbs",
"chars": 211,
"preview": "<div class=\"row\" id=\"warning\" {{#unless message}}style=\"display: none;\"{{/unless}}>\n <div class=\"col-md-12\">\n "
},
{
"path": "nsqadmin/static.go",
"chars": 182,
"preview": "package nsqadmin\n\nimport (\n\t\"embed\"\n)\n\n//go:embed static/build\nvar static embed.FS\n\nfunc staticAsset(name string) ([]byt"
},
{
"path": "nsqadmin/test/ca.key",
"chars": 561,
"preview": "-----BEGIN RSA PRIVATE KEY-----\nProc-Type: 4,ENCRYPTED\nDEK-Info: DES-EDE3-CBC,D6208487450BDF1C\n\ntucU/4j6agGQlW60D8V3Zr/Q"
},
{
"path": "nsqadmin/test/ca.pem",
"chars": 522,
"preview": "-----BEGIN CERTIFICATE-----\nMIIBVDCB/6ADAgECAgkAvHG4Z/7nX/gwDQYJKoZIhvcNAQEFBQAwDTELMAkGA1UE\nAwwCY2EwHhcNMTcwOTE2MTc0NDE"
},
{
"path": "nsqadmin/test/ca.srl",
"chars": 17,
"preview": "91418D04995922E7\n"
},
{
"path": "nsqadmin/test/cert.pem",
"chars": 1598,
"preview": "-----BEGIN CERTIFICATE-----\nMIIEbjCCA1agAwIBAgIJAK6x7y6AwBmLMA0GCSqGSIb3DQEBBQUAMIGAMQswCQYD\nVQQGEwJVUzERMA8GA1UECBMITmV"
},
{
"path": "nsqadmin/test/client.key",
"chars": 1675,
"preview": "-----BEGIN RSA PRIVATE KEY-----\nMIIEowIBAAKCAQEAuMz5IJ01Wqaj2qJAU7o2aoc2iP1BRhTba+cWrsjbZA+6kXAk\n19Dlz/nNK8gxsBMl3rzTGWX"
},
{
"path": "nsqadmin/test/client.pem",
"chars": 1029,
"preview": "-----BEGIN CERTIFICATE-----\nMIICyzCCAnWgAwIBAgIJAJFBjQSZWSLnMA0GCSqGSIb3DQEBCwUAMA0xCzAJBgNV\nBAMMAmNhMB4XDTE3MDkxNjE3NDQ"
},
{
"path": "nsqadmin/test/client.req",
"chars": 1050,
"preview": "-----BEGIN CERTIFICATE REQUEST-----\nMIICzTCCAbUCAQAwgYcxIzAhBgkqhkiG9w0BCQEWFG1yZWlmZXJzb25AZ21haWwu\nY29tMQswCQYDVQQGEwJ"
},
{
"path": "nsqadmin/test/key.pem",
"chars": 1675,
"preview": "-----BEGIN RSA PRIVATE KEY-----\nMIIEogIBAAKCAQEAnX0KB+svwy+yHU2qggz/EaGgcraKShagKo+9M9y5HLM852ng\nk5c+t+tJJbx3N954Wr1FXBu"
},
{
"path": "nsqadmin/test/server.key",
"chars": 1675,
"preview": "-----BEGIN RSA PRIVATE KEY-----\nMIIEowIBAAKCAQEAuYs7NerYpXWq/IPaBGriHobhIs/i4AJHra7QhZ5N/EfBCb0N\n6K3tI4GRHMZ4lB0HwINWmxa"
},
{
"path": "nsqadmin/test/server.pem",
"chars": 1054,
"preview": "-----BEGIN CERTIFICATE-----\nMIIC3jCCAoigAwIBAgIJAJFBjQSZWSLmMA0GCSqGSIb3DQEBCwUAMA0xCzAJBgNV\nBAMMAmNhMB4XDTE3MDkxNjE3NDQ"
},
{
"path": "nsqadmin/test/server.req",
"chars": 1050,
"preview": "-----BEGIN CERTIFICATE REQUEST-----\nMIICzTCCAbUCAQAwgYcxIzAhBgkqhkiG9w0BCQEWFG1yZWlmZXJzb25AZ21haWwu\nY29tMQswCQYDVQQGEwJ"
},
{
"path": "nsqd/README.md",
"chars": 141,
"preview": "## nsqd\n\n`nsqd` is the daemon that receives, queues, and delivers messages to clients.\n\nRead the [docs](https://nsq.io/c"
},
{
"path": "nsqd/backend_queue.go",
"chars": 286,
"preview": "package nsqd\n\n// BackendQueue represents the behavior for the secondary message\n// storage system\ntype BackendQueue inte"
},
{
"path": "nsqd/buffer_pool.go",
"chars": 272,
"preview": "package nsqd\n\nimport (\n\t\"bytes\"\n\t\"sync\"\n)\n\nvar bp sync.Pool\n\nfunc init() {\n\tbp.New = func() interface{} {\n\t\treturn &byte"
},
{
"path": "nsqd/channel.go",
"chars": 16010,
"preview": "package nsqd\n\nimport (\n\t\"container/heap\"\n\t\"errors\"\n\t\"fmt\"\n\t\"math\"\n\t\"strings\"\n\t\"sync\"\n\t\"sync/atomic\"\n\t\"time\"\n\n\t\"github.co"
},
{
"path": "nsqd/channel_test.go",
"chars": 6771,
"preview": "package nsqd\n\nimport (\n\t\"fmt\"\n\t\"io\"\n\t\"net/http\"\n\t\"os\"\n\t\"strconv\"\n\t\"testing\"\n\t\"time\"\n\n\t\"github.com/nsqio/nsq/internal/tes"
},
{
"path": "nsqd/client_v2.go",
"chars": 18686,
"preview": "package nsqd\n\nimport (\n\t\"bufio\"\n\t\"compress/flate\"\n\t\"crypto/tls\"\n\t\"fmt\"\n\t\"net\"\n\t\"strings\"\n\t\"sync\"\n\t\"sync/atomic\"\n\t\"time\"\n"
},
{
"path": "nsqd/dqname.go",
"chars": 270,
"preview": "//go:build !windows\n// +build !windows\n\npackage nsqd\n\nfunc getBackendName(topicName, channelName string) string {\n\t// ba"
},
{
"path": "nsqd/dqname_windows.go",
"chars": 317,
"preview": "//go:build windows\n// +build windows\n\npackage nsqd\n\n// On Windows, file names cannot contain colons.\nfunc getBackendName"
},
{
"path": "nsqd/dummy_backend_queue.go",
"chars": 554,
"preview": "package nsqd\n\ntype dummyBackendQueue struct {\n\treadChan chan []byte\n}\n\nfunc newDummyBackendQueue() BackendQueue {\n\tretur"
},
{
"path": "nsqd/guid.go",
"chars": 2069,
"preview": "package nsqd\n\n// the core algorithm here was borrowed from:\n// Blake Mizerany's `noeqd` https://github.com/bmizerany/noe"
},
{
"path": "nsqd/guid_test.go",
"chars": 728,
"preview": "package nsqd\n\nimport (\n\t\"testing\"\n\t\"unsafe\"\n)\n\nfunc BenchmarkGUIDCopy(b *testing.B) {\n\tsource := make([]byte, 16)\n\tvar d"
},
{
"path": "nsqd/http.go",
"chars": 21180,
"preview": "package nsqd\n\nimport (\n\t\"bufio\"\n\t\"bytes\"\n\t\"encoding/json\"\n\t\"fmt\"\n\t\"io\"\n\t\"net\"\n\t\"net/http\"\n\t\"net/http/pprof\"\n\t\"net/url\"\n\t"
},
{
"path": "nsqd/http_test.go",
"chars": 27649,
"preview": "package nsqd\n\nimport (\n\t\"bytes\"\n\t\"crypto/tls\"\n\t\"encoding/json\"\n\t\"fmt\"\n\t\"io\"\n\t\"net\"\n\t\"net/http\"\n\t\"os\"\n\t\"runtime\"\n\t\"strcon"
},
{
"path": "nsqd/in_flight_pqueue.go",
"chars": 1642,
"preview": "package nsqd\n\ntype inFlightPqueue []*Message\n\nfunc newInFlightPqueue(capacity int) inFlightPqueue {\n\treturn make(inFligh"
},
{
"path": "nsqd/in_flight_pqueue_test.go",
"chars": 1539,
"preview": "package nsqd\n\nimport (\n\t\"fmt\"\n\t\"math/rand\"\n\t\"sort\"\n\t\"testing\"\n\n\t\"github.com/nsqio/nsq/internal/test\"\n)\n\nfunc TestPriorit"
},
{
"path": "nsqd/logger.go",
"chars": 355,
"preview": "package nsqd\n\nimport (\n\t\"github.com/nsqio/nsq/internal/lg\"\n)\n\ntype Logger lg.Logger\n\nconst (\n\tLOG_DEBUG = lg.DEBUG\n\tLOG_"
},
{
"path": "nsqd/lookup.go",
"chars": 5152,
"preview": "package nsqd\n\nimport (\n\t\"bytes\"\n\t\"encoding/json\"\n\t\"net\"\n\t\"os\"\n\t\"strconv\"\n\t\"time\"\n\n\t\"github.com/nsqio/go-nsq\"\n\t\"github.co"
},
{
"path": "nsqd/lookup_peer.go",
"chars": 3776,
"preview": "package nsqd\n\nimport (\n\t\"encoding/binary\"\n\t\"fmt\"\n\t\"io\"\n\t\"net\"\n\t\"time\"\n\n\t\"github.com/nsqio/go-nsq\"\n\t\"github.com/nsqio/nsq"
},
{
"path": "nsqd/message.go",
"chars": 2297,
"preview": "package nsqd\n\nimport (\n\t\"encoding/binary\"\n\t\"fmt\"\n\t\"io\"\n\t\"time\"\n)\n\nconst (\n\tMsgIDLength = 16\n\tminValidMsgLength = M"
},
{
"path": "nsqd/nsqd.go",
"chars": 19390,
"preview": "package nsqd\n\nimport (\n\t\"context\"\n\t\"crypto/tls\"\n\t\"crypto/x509\"\n\t\"encoding/json\"\n\t\"errors\"\n\t\"fmt\"\n\t\"log\"\n\t\"math/rand\"\n\t\"n"
},
{
"path": "nsqd/nsqd_test.go",
"chars": 12198,
"preview": "package nsqd\n\nimport (\n\t\"encoding/json\"\n\t\"errors\"\n\t\"fmt\"\n\t\"io/fs\"\n\t\"net\"\n\t\"os\"\n\t\"strconv\"\n\t\"sync/atomic\"\n\t\"testing\"\n\t\"ti"
},
{
"path": "nsqd/options.go",
"chars": 6237,
"preview": "package nsqd\n\nimport (\n\t\"crypto/md5\"\n\t\"crypto/tls\"\n\t\"hash/crc32\"\n\t\"io\"\n\t\"log\"\n\t\"os\"\n\t\"time\"\n\n\t\"github.com/nsqio/nsq/inte"
},
{
"path": "nsqd/protocol_v2.go",
"chars": 32645,
"preview": "package nsqd\n\nimport (\n\t\"bytes\"\n\t\"encoding/binary\"\n\t\"encoding/json\"\n\t\"errors\"\n\t\"fmt\"\n\t\"io\"\n\t\"math/rand\"\n\t\"net\"\n\t\"sync/at"
},
{
"path": "nsqd/protocol_v2_test.go",
"chars": 55733,
"preview": "package nsqd\n\nimport (\n\t\"bufio\"\n\t\"bytes\"\n\t\"compress/flate\"\n\t\"crypto/tls\"\n\t\"encoding/json\"\n\t\"errors\"\n\t\"fmt\"\n\t\"io\"\n\t\"math\""
},
{
"path": "nsqd/protocol_v2_unixsocket_test.go",
"chars": 49569,
"preview": "package nsqd\n\nimport (\n\t\"bufio\"\n\t\"bytes\"\n\t\"compress/flate\"\n\t\"crypto/tls\"\n\t\"encoding/json\"\n\t\"errors\"\n\t\"fmt\"\n\t\"math\"\n\t\"mat"
},
{
"path": "nsqd/stats.go",
"chars": 6390,
"preview": "package nsqd\n\nimport (\n\t\"runtime\"\n\t\"sort\"\n\t\"sync/atomic\"\n\n\t\"github.com/nsqio/nsq/internal/quantile\"\n)\n\ntype Stats struct"
},
{
"path": "nsqd/stats_test.go",
"chars": 4217,
"preview": "package nsqd\n\nimport (\n\t\"encoding/json\"\n\t\"fmt\"\n\t\"os\"\n\t\"strconv\"\n\t\"sync\"\n\t\"testing\"\n\t\"time\"\n\n\t\"github.com/golang/snappy\"\n"
},
{
"path": "nsqd/statsd.go",
"chars": 5542,
"preview": "package nsqd\n\nimport (\n\t\"fmt\"\n\t\"math\"\n\t\"net\"\n\t\"strings\"\n\t\"time\"\n\n\t\"github.com/nsqio/nsq/internal/statsd\"\n\t\"github.com/ns"
},
{
"path": "nsqd/tcp.go",
"chars": 1607,
"preview": "package nsqd\n\nimport (\n\t\"io\"\n\t\"net\"\n\t\"sync\"\n\n\t\"github.com/nsqio/nsq/internal/protocol\"\n)\n\nconst (\n\ttypeConsumer = iota\n\t"
},
{
"path": "nsqd/test/cert.sh",
"chars": 1298,
"preview": "#!/bin/bash\n# ./cert.sh foo@foo.com 127.0.0.1\n# Found: https://gist.github.com/ncw/9253562#file-makecert-sh\n\nif [ \"$1\" ="
},
{
"path": "nsqd/test/certs/ca.key",
"chars": 561,
"preview": "-----BEGIN RSA PRIVATE KEY-----\nProc-Type: 4,ENCRYPTED\nDEK-Info: DES-EDE3-CBC,D6208487450BDF1C\n\ntucU/4j6agGQlW60D8V3Zr/Q"
},
{
"path": "nsqd/test/certs/ca.pem",
"chars": 522,
"preview": "-----BEGIN CERTIFICATE-----\nMIIBVDCB/6ADAgECAgkAvHG4Z/7nX/gwDQYJKoZIhvcNAQEFBQAwDTELMAkGA1UE\nAwwCY2EwHhcNMTcwOTE2MTc0NDE"
},
{
"path": "nsqd/test/certs/ca.srl",
"chars": 17,
"preview": "91418D04995922E7\n"
},
{
"path": "nsqd/test/certs/cert.pem",
"chars": 1598,
"preview": "-----BEGIN CERTIFICATE-----\nMIIEbjCCA1agAwIBAgIJAK6x7y6AwBmLMA0GCSqGSIb3DQEBBQUAMIGAMQswCQYD\nVQQGEwJVUzERMA8GA1UECBMITmV"
},
{
"path": "nsqd/test/certs/client.key",
"chars": 1675,
"preview": "-----BEGIN RSA PRIVATE KEY-----\nMIIEowIBAAKCAQEAuMz5IJ01Wqaj2qJAU7o2aoc2iP1BRhTba+cWrsjbZA+6kXAk\n19Dlz/nNK8gxsBMl3rzTGWX"
},
{
"path": "nsqd/test/certs/client.pem",
"chars": 1029,
"preview": "-----BEGIN CERTIFICATE-----\nMIICyzCCAnWgAwIBAgIJAJFBjQSZWSLnMA0GCSqGSIb3DQEBCwUAMA0xCzAJBgNV\nBAMMAmNhMB4XDTE3MDkxNjE3NDQ"
},
{
"path": "nsqd/test/certs/client.req",
"chars": 1050,
"preview": "-----BEGIN CERTIFICATE REQUEST-----\nMIICzTCCAbUCAQAwgYcxIzAhBgkqhkiG9w0BCQEWFG1yZWlmZXJzb25AZ21haWwu\nY29tMQswCQYDVQQGEwJ"
},
{
"path": "nsqd/test/certs/key.pem",
"chars": 1675,
"preview": "-----BEGIN RSA PRIVATE KEY-----\nMIIEogIBAAKCAQEAnX0KB+svwy+yHU2qggz/EaGgcraKShagKo+9M9y5HLM852ng\nk5c+t+tJJbx3N954Wr1FXBu"
},
{
"path": "nsqd/test/certs/server.key",
"chars": 1675,
"preview": "-----BEGIN RSA PRIVATE KEY-----\nMIIEowIBAAKCAQEAuYs7NerYpXWq/IPaBGriHobhIs/i4AJHra7QhZ5N/EfBCb0N\n6K3tI4GRHMZ4lB0HwINWmxa"
},
{
"path": "nsqd/test/certs/server.pem",
"chars": 1054,
"preview": "-----BEGIN CERTIFICATE-----\nMIIC3jCCAoigAwIBAgIJAJFBjQSZWSLmMA0GCSqGSIb3DQEBCwUAMA0xCzAJBgNV\nBAMMAmNhMB4XDTE3MDkxNjE3NDQ"
},
{
"path": "nsqd/test/certs/server.req",
"chars": 1050,
"preview": "-----BEGIN CERTIFICATE REQUEST-----\nMIICzTCCAbUCAQAwgYcxIzAhBgkqhkiG9w0BCQEWFG1yZWlmZXJzb25AZ21haWwu\nY29tMQswCQYDVQQGEwJ"
},
{
"path": "nsqd/test/openssl.conf",
"chars": 617,
"preview": "[req]\ndistinguished_name = req_distinguished_name\n\n[req_distinguished_name]\n\n[ca]\nbasicConstraints = critical, CA:true\ns"
},
{
"path": "nsqd/topic.go",
"chars": 11583,
"preview": "package nsqd\n\nimport (\n\t\"errors\"\n\t\"strings\"\n\t\"sync\"\n\t\"sync/atomic\"\n\t\"time\"\n\n\t\"github.com/nsqio/go-diskqueue\"\n\t\"github.co"
},
{
"path": "nsqd/topic_test.go",
"chars": 6090,
"preview": "package nsqd\n\nimport (\n\t\"errors\"\n\t\"fmt\"\n\t\"io\"\n\t\"net/http\"\n\t\"os\"\n\t\"runtime\"\n\t\"strconv\"\n\t\"testing\"\n\t\"time\"\n\n\t\"github.com/n"
},
{
"path": "nsqlookupd/README.md",
"chars": 207,
"preview": "## nsqlookupd\n\n`nsqlookupd` is the daemon that manages topology metadata and serves client requests to\ndiscover the loca"
},
{
"path": "nsqlookupd/client_v1.go",
"chars": 249,
"preview": "package nsqlookupd\n\nimport (\n\t\"net\"\n)\n\ntype ClientV1 struct {\n\tnet.Conn\n\tpeerInfo *PeerInfo\n}\n\nfunc NewClientV1(conn net"
},
{
"path": "nsqlookupd/http.go",
"chars": 11438,
"preview": "package nsqlookupd\n\nimport (\n\t\"fmt\"\n\t\"net/http\"\n\t\"net/http/pprof\"\n\t\"sync/atomic\"\n\n\t\"github.com/julienschmidt/httprouter\""
}
]
// ... and 11 more files (download for full content)
About this extraction
This page contains the full source code of the nsqio/nsq GitHub repository, extracted and formatted as plain text for AI agents and large language models (LLMs). The extraction includes 211 files (1.2 MB), approximately 373.1k tokens, and a symbol index with 1336 extracted functions, classes, methods, constants, and types. Use this with OpenClaw, Claude, ChatGPT, Cursor, Windsurf, or any other AI tool that accepts text input. You can copy the full output to your clipboard or download it as a .txt file.
Extracted by GitExtract — free GitHub repo to text converter for AI. Built by Nikandr Surkov.