Showing preview only (409K chars total). Download the full file or copy to clipboard to get everything.
Repository: nats-io/nats.ex
Branch: main
Commit: b6d2f37e8901
Files: 85
Total size: 384.2 KB
Directory structure:
gitextract_y72cyuck/
├── .dialyzer_ignore.exs
├── .formatter.exs
├── .github/
│ └── workflows/
│ └── CI.yml
├── .gitignore
├── CHANGELOG.md
├── LICENSE.txt
├── MAINTAINERS.md
├── README.md
├── bench/
│ ├── client.exs
│ ├── kv_consume.exs
│ ├── parse.exs
│ ├── publish.exs
│ ├── request.exs
│ ├── request_multi.exs
│ ├── server.exs
│ └── service_bench.exs
├── dependencies.md
├── docs/
│ └── js/
│ ├── guides/
│ │ ├── broadway.md
│ │ ├── managing.md
│ │ └── push_based_consumer.md
│ └── introduction/
│ ├── getting_started.md
│ └── overview.md
├── lib/
│ ├── gnat/
│ │ ├── command.ex
│ │ ├── connection_supervisor.ex
│ │ ├── consumer_supervisor.ex
│ │ ├── handshake.ex
│ │ ├── jetstream/
│ │ │ ├── api/
│ │ │ │ ├── consumer.ex
│ │ │ │ ├── kv/
│ │ │ │ │ ├── entry.ex
│ │ │ │ │ └── watcher.ex
│ │ │ │ ├── kv.ex
│ │ │ │ ├── message.ex
│ │ │ │ ├── object/
│ │ │ │ │ └── meta.ex
│ │ │ │ ├── object.ex
│ │ │ │ ├── stream.ex
│ │ │ │ └── util.ex
│ │ │ ├── jetstream.ex
│ │ │ ├── pager.ex
│ │ │ ├── pull_consumer/
│ │ │ │ ├── connection_options.ex
│ │ │ │ └── server.ex
│ │ │ └── pull_consumer.ex
│ │ ├── parsec.ex
│ │ ├── server.ex
│ │ └── services/
│ │ ├── server.ex
│ │ ├── service.ex
│ │ ├── service_responder.ex
│ │ └── wire_protocol.ex
│ └── gnat.ex
├── mix.exs
├── scripts/
│ └── cluster/
│ ├── cluster.sh
│ ├── driver.exs
│ ├── n1.conf
│ ├── n2.conf
│ └── n3.conf
└── test/
├── command_test.exs
├── fixtures/
│ ├── ca.pem
│ ├── client-cert.pem
│ ├── client-key.pem
│ ├── nkey_config
│ ├── nkey_seed
│ ├── server-cert.pem
│ └── server-key.pem
├── gnat/
│ ├── consumer_supervisor_test.exs
│ ├── handshake_test.exs
│ ├── parsec_property_test.exs
│ └── parsec_test.exs
├── gnat_property_test.exs
├── gnat_test.exs
├── jetstream/
│ ├── api/
│ │ ├── consumer_doc_test.exs
│ │ ├── consumer_test.exs
│ │ ├── kv/
│ │ │ ├── entry_test.exs
│ │ │ └── watcher_test.exs
│ │ ├── kv_test.exs
│ │ ├── object_test.exs
│ │ ├── stream_doc_test.exs
│ │ └── stream_test.exs
│ ├── message_test.exs
│ └── pager_test.exs
├── pull_consumer/
│ ├── batch_test.exs
│ ├── connectivity_test.exs
│ ├── ephemeral_test.exs
│ ├── status_messages_test.exs
│ └── using_macro_test.exs
├── support/
│ ├── conn_case.ex
│ └── generators.ex
└── test_helper.exs
================================================
FILE CONTENTS
================================================
================================================
FILE: .dialyzer_ignore.exs
================================================
[
]
================================================
FILE: .formatter.exs
================================================
[
inputs: ["{.formatter,mix}.exs", "{lib,test}/**/*.{ex,exs}"]
]
================================================
FILE: .github/workflows/CI.yml
================================================
name: CI
on:
push:
branches: [main]
pull_request:
branches: [main]
schedule:
- cron: "0 0 1 */1 *"
permissions:
contents: read
jobs:
test:
name: Elixir ${{ matrix.pair.elixir }} / OTP ${{ matrix.pair.otp }} / NATS ${{ matrix.pair.nats }}
runs-on: ubuntu-latest
strategy:
matrix:
pair:
- otp: "25"
elixir: "1.14"
nats: "2.10.0"
- otp: "26"
elixir: "1.16"
nats: "2.10.24"
- otp: "27"
elixir: "1.17"
nats: "2.10.24"
- otp: "27"
elixir: "1.18"
nats: "2.10.24"
- otp: "28"
elixir: "1.18"
nats: "2.11.11"
- otp: "28"
elixir: "1.18"
nats: "2.10.24"
# the main pair handles things like format checking and running dialyzer
- main: true
otp: "28"
elixir: "1.19"
nats: "2.10.24"
env:
MIX_ENV: test
steps:
- name: Checkout
uses: actions/checkout@v4
- name: Set up Elixir
uses: erlef/setup-beam@v1
with:
elixir-version: ${{ matrix.pair.elixir }}
otp-version: ${{ matrix.pair.otp }}
- name: Start NATS Jetstream
run: docker run --rm -d --network host nats:${{ matrix.pair.nats }} -js
- name: Restore deps cache
uses: actions/cache@v4
with:
path: |
deps
_build
key: deps-${{ matrix.pair.otp }}-${{ matrix.pair.elixir }}-${{ hashFiles('mix.lock') }}
- name: Install package dependencies
run: mix deps.get
- name: Check for valid formatting
if: matrix.pair.main
run: mix format --check-formatted
- name: Run unit tests
run: mix test --color
- name: Run NATS 2.11 specific tests
if: ${{ matrix.pair.nats == '2.11.11' }}
run: mix test --only message_ttl
- name: Cache Dialyzer PLTs
if: matrix.pair.main
uses: actions/cache@v4
with:
path: |
priv/plts/*.plt
priv/plts/*.plt.hash
key: dialyzer-${{ matrix.pair.otp }}-${{ matrix.pair.elixir }}-${{ hashFiles('mix.lock') }}
restore-keys: |
dialyzer-${{ matrix.pair.otp }}-${{ matrix.pair.elixir }}-
- name: Run Dialyzer
if: matrix.pair.main
env:
MIX_ENV: dev
run: |
mix dialyzer
================================================
FILE: .gitignore
================================================
# The directory Mix will write compiled artifacts to.
/_build
# If you run "mix test --cover", coverage assets end up here.
/cover
# The directory Mix downloads your dependencies sources to.
/deps
/tmp
# Where 3rd-party dependencies like ExDoc output generated docs.
/doc
# Dialyzer PLT files
priv/plts/
# Ignore .fetch files in case you like to edit your project deps locally.
/.fetch
# If the VM crashes, it generates a dump, let's ignore it too.
erl_crash.dump
# Also ignore archive artifacts (built via "mix archive.build").
*.ez
# Do not commit the counterexamples stored during property testing
/test/counter_examples
# ASDF file to for specifying which erlang/elixir version to use for local development
.tool-versions
# Ignore Cluster details
scripts/cluster/data
scripts/cluster/logs
scripts/cluster/pids
================================================
FILE: CHANGELOG.md
================================================
# Changelog
## 1.14
* Add `PullConsumer.handle_connected/2` optional callback to get consumer info
* Add `PullConsumer.handle_status/2` optional callback to observe status messages
* Added support for `batch_size` option in PullConsumer options to pull messages
and acknowledge them in batches.
* Add `Gnat.Jetstream.API.KV.Entry` with `from_message/2` for parsing a raw
NATS message from a KV bucket's underlying stream into a structured entry
(operation, key, value, revision, created, delta). Intended to be shared
between the built-in `KV.Watcher` and user-supplied `PullConsumer`
implementations (e.g. caches that need to detect when they are caught up
with the stream). Returns `:ignore` for messages that are not KV records.
* `KV.Watcher` now uses `KV.Entry` internally; its public callback API is
unchanged. The push consumer it creates now enables server-driven flow
control and a 5s idle heartbeat (matching nats.go's ordered-consumer
defaults), so slow handlers apply backpressure instead of being dropped
as slow consumers.
* **Behavior change (bugfix):** `PullConsumer` no longer forwards JetStream
informational status messages (e.g. `100` idle heartbeat, `409` leadership
change) to `c:handle_message/2`. These are not stream records and cannot
be acked. In single-message mode the consumer now drops them and re-issues
a pull request.
* Add an optional `c:handle_status/2` callback to `Gnat.Jetstream.PullConsumer`
for users who want to observe status messages (e.g. log on `409`).
## 1.11
* Allow clients to force authentication without server auth_required by @mmmries in #205
* Support ephemeral consumers and auto-cleanup consumers in the PullConsumer module by @mmmries in #202
* Implement regularly scheduled PING/PONG health check by @mmmries in #200
* Add Erlang 28 with Elixir 1.18 to the build matrix by @davydog187 in #201
* Use Pager module for KV.contents by @mmmries in #198
* Hint `:timeout` option in KV's typespec by @rixmann in #193
## 1.10
* Clarify authentication setup during test by @davydog187 in #187
* Test on Elixir 1.18 and NATS 2.10.24 by @davydog187 in #188
* Gnat.Jetstream.API.KV.info/3 by @davydog187 in #189
* Remove function_exported? check for Keyword.validate!/2 by @davydog187 in #186
* Tiny optimization to KV.list_buckets/1 by @davydog187 in #185
* make KV-watcher emit :key_added events when the message has a header by @rixmann in #191
* add :compression to stream attributes by @rixmann in #192
* fix: unknown field domain in Stream.create (#194) by @c0deaddict
* feat: add jetstream message metadata helper (#197) by @c0deaddict
* fix: deliver policy (#196) by @c0deaddict
## 1.9
* Housecleaning by @mmmries in #176
* switch to charlist sigils
* update to newest nkeys
* require elixir 1.14 and erlang 25+
* Fix incorrect useage of charlist by @davydog187 in #179
* Soft deprecate is_kv_bucket_stream?/1 in favor of kv_bucket_stream?/1 by @davydog187 in #183
* Clean up examples in KV by @davydog187 in #181
* Document options for Gnat.Jetstream.API.KV by @davydog187 in #180
## 1.8
* Integrated the jetstream functionality into this client directly https://github.com/nats-io/nats.ex/pull/146
* Add ability to list KV buckets https://github.com/nats-io/nats.ex/pull/152
* Improve CI Reliability https://github.com/nats-io/nats.ex/pull/154
* Bugfix to treat no streams as an empty list rather than a null https://github.com/nats-io/nats.ex/pull/155
* Added supported for `allow_direct` and `mirror_direct` attributes of streams https://github.com/nats-io/nats.ex/pull/161
* Added support for `discard_new_per_subject` attribute of streams https://github.com/nats-io/nats.ex/pull/163
* Added support for `Object.list_buckets` https://github.com/nats-io/nats.ex/pull/169
## 1.7
* Added support for the NATS [services API](https://github.com/nats-io/nats.go/blob/main/micro/README.md), letting developers participate in service discovery and stats https://github.com/nats-io/nats.ex/pull/141
* A bugfix to remove the queue_group from a service config and some optimization for the services API https://github.com/nats-io/nats.ex/pull/145
## 1.6
* added the `no_responders` behavior https://github.com/nats-io/nats.ex/pull/137
## 1.5
* add the `inbox_prefix` option https://github.com/nats-io/nats.ex/pull/121
* add the `Gnat.server_info/1` function https://github.com/nats-io/nats.ex/pull/124
* fix header parsing issue https://github.com/nats-io/nats.ex/pull/125
## 1.4
* add the `Gnat.request_multi/4` function https://github.com/nats-io/nats.ex/pull/120
* add elixir 1.13 to the test matrix
## 1.3
* adding support for sending and receiving headers https://github.com/nats-io/nats.ex/pull/116
## 1.2
* `Gnat.Server` behaviour with support in the `ConsumerSupervisor` https://github.com/nats-io/nats.ex/compare/1b1adc85e4b28231218ef87c7fc3445fce854377...b24a7e14325b51fbb93fde7e3d891d18b4fa8afb
* avoid logging sensitive credentials https://github.com/nats-io/nats.ex/pull/105
* deprecate Gnat.ping, improved typespecs https://github.com/nats-io/nats.ex/pull/103
* relax the version constraint on nimble_parsec https://github.com/nats-io/nats.ex/issues/112
## 1.1
* add support for nkeys and NGS https://github.com/nats-io/nats.ex/pull/101
* Fix supervisor ConsumerSuperivsor crash https://github.com/nats-io/nats.ex/pull/96
## 1.0
* Make supervisors officially supported https://github.com/nats-io/nats.ex/pull/96
## 0.7.0
* update to telemetry 0.4 https://github.com/nats-io/nats.ex/pull/86 and https://github.com/nats-io/nats.ex/pull/87
* support for token authentication https://github.com/nats-io/nats.ex/pull/92
* support elixir 1.9 https://github.com/nats-io/nats.ex/pull/93
## 0.6.0
* Dropped support for Erlang < 19 and Elixir <= 1.5
* Added Telemetry to the project (thanks @rubysolo)
* Switched to nimble_parsec for parsing
* Updated benchmarking/performance information. We can now do 170k requests per second on a 16-core server.
* Fixed a bug around re-subscribing for the `ConsumerSupervisor`
* Pass `sid` when delivering message (thanks @entone)
* Documentation fixes from @deini and @johannestroeger
## 0.5.0
* Dropped support for Elixir 1.4 and OTP 18 releases. You will need to use Elixir 1.5+ and OTP 19+.
* Switched to running our tests against gnatsd `1.3.0`
================================================
FILE: LICENSE.txt
================================================
Copyright 2017 Michael Ries
Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
================================================
FILE: MAINTAINERS.md
================================================
# Maintainers
Maintainership is on a per project basis.
### Maintainers
- Colin Sullivan <colin@nats.io> [@ColinSullivan1](https://github.com/ColinSullivan1)
- Michael Ries <riesmmm@gmail.com> [@mmmries](https://github.com/mmmries)
- Kevin Hoffman <alothien@gmail.com> [@autodidaddict](https//github.com/autodidaddict)
================================================
FILE: README.md
================================================
[](https://hex.pm/packages/gnat)
[](https://hex.pm/packages/gnat)
[](https://hex.pm/packages/gnat)
[](https://github.com/nats-io/nats.ex)
# Gnat
A [nats.io](https://nats.io/) client for Elixir.
The goals of the project are resiliency, performance, and ease of use.
> Hex documentation available here: https://hex.pm/packages/gnat
## Usage
``` elixir
{:ok, gnat} = Gnat.start_link(%{host: "127.0.0.1", port: 4222})
# Or if the server requires TLS you can start a connection with:
# {:ok, gnat} = Gnat.start_link(%{host: "127.0.0.1", port: 4222, tls: true})
{:ok, subscription} = Gnat.sub(gnat, self(), "pawnee.*")
:ok = Gnat.pub(gnat, "pawnee.news", "Leslie Knope recalled from city council (Jammed)")
receive do
{:msg, %{body: body, topic: "pawnee.news", reply_to: nil}} ->
IO.puts(body)
end
```
## Authentication
``` elixir
# with user and password
{:ok, gnat} = Gnat.start_link(%{host: "127.0.0.1", port: 4222, username: "joe", password: "123", auth_required: true})
# with token
{:ok, gnat} = Gnat.start_link(%{host: "127.0.0.1", port: 4222, token: "secret", auth_required: true})
# with an nkey seed
{:ok, gnat} = Gnat.start_link(%{host: "127.0.0.1", port: 4222, nkey_seed: "SUAM...", auth_required: true})
# with decentralized user credentials (JWT)
{:ok, gnat} = Gnat.start_link(%{host: "127.0.0.1", port: 4222, nkey_seed: "SUAM...", jwt: "eyJ0eX...", auth_required: true})
# connect to NGS with JWT
{:ok, gnat} = Gnat.start_link(%{host: "connect.ngs.global", tls: true, jwt: "ey...", nkey_seed: "SUAM..."})
```
## TLS Connections
[NATS Server](https://github.com/nats-io/nats-server) is often configured to accept or require TLS connections.
In order to connect to these clusters you'll want to pass some extra TLS settings to your `Gnat` connection.
``` elixir
# using a basic TLS connection
{:ok, gnat} = Gnat.start_link(%{host: "127.0.0.1", port: 4222, tls: true})
# Passing a Client Certificate for verification
{:ok, gnat} = Gnat.start_link(%{tls: true, ssl_opts: [certfile: "client-cert.pem", keyfile: "client-key.pem"]})
```
## Resiliency
If you would like to stay connected to a cluster of nats servers, you should consider using `Gnat.ConnectionSupervisor` .
This can be added to your supervision tree in your project and will handle automatically re-connecting to the cluster.
For long-lived subscriptions consider using `Gnat.ConsumerSupervisor` .
This can also be added to your supervision tree and use a supervised connection to re-establish a subscription.
It also handles details like handling each message in a supervised process so you isolate failures and get OTP logs when an unexpected error occurs.
## Services
If you supply a module that implements the `Gnat.Services.Server` behavior and the `service_definition`
configuration field to a `Gnat.ConsumerSupervisor`, then this client will automatically take care
of exposing the service to discovery, responding to pings, and maintaining and exposing statistics like request and error counts, and processing times.
## Instrumentation
Gnat uses [telemetry](https://hex.pm/packages/telemetry) to make instrumentation data available to clients.
If you want to record metrics around the number of messages or latency of message publishes, subscribes, requests, etc you can do the following in your project:
``` elixir
iex(1)> metrics_function = fn(event_name, measurements, event_meta, config) ->
IO.inspect([event_name, measurements, event_meta, config])
:ok
end
#Function<4.128620087/4 in :erl_eval.expr/5>
iex(2)> names = [[:gnat, :pub], [:gnat, :sub], [:gnat, :message_received], [:gnat, :request], [:gnat, :unsub]]
[
[:gnat, :pub],
[:gnat, :sub],
[:gnat, :message_received],
[:gnat, :request],
[:gnat, :unsub],
[:gnat, :service_request],
[:gnat, :service_error]
]
iex(3)> :telemetry.attach_many("my listener", names, metrics_function, %{my_config: true})
:ok
iex(4)> {:ok, gnat} = Gnat.start_link()
{:ok, #PID<0.203.0>}
iex(5)> Gnat.sub(gnat, self(), "topic")
[[:gnat, :sub], %{latency: 128000}, %{topic: "topic"}, %{my_config: true}]
{:ok, 1}
iex(6)> Gnat.pub(gnat, "topic", "ohai")
[[:gnat, :pub], %{latency: 117000}, %{topic: "topic"}, %{my_config: true}]
[[:gnat, :message_received], %{count: 1}, %{topic: "topic"}, %{my_config: true}]
:ok
```
The `pub` , `sub` , `request` , and `unsub` events all report the latency of those respective calls.
The `message_received` event reports a number of messages like `%{count: 1}` because there isn't a good latency metric to report. Any microservices managed by a consumer supervisor will also report `service_request` and `service_error`. In addition to the `:topic` metadata, microservices will also include `:endpoint` and `:group` (which can be `nil`) in their telemetry reports.
All of the events (except `unsub` ) include metadata with a `:topic` key so you can split your metrics by topic.
## Benchmarks
Part of the motivation for building this library is to get better performance.
To this end, there is a `bench` branch on this project which includes a `server.exs` and `client.exs` that can be used for benchmarking various scenarios.
As of this commit, the [latest benchmark on a 16-core server](https://gist.github.com/mmmries/08fe44fdd47a6f8838936f41170f270a) shows that you can make 170k+ req/sec or up to 192MB/sec.
The `bench/*.exs` files also contain some straight-line single-CPU performance tests.
As of this commit my 2018 MacBook pro shows.
| | ips | average | deviation | median |
| ------------- | -------- | --------- | --------- | ------ |
| parse-128 | 487.67 K | 2.19 μs | ±1701.54% | 2 μs |
| pub - 128 | 96.37 K | 10.38 μs | ±102.94% | 10 μs |
| req-reply-128 | 8.32 K | 120.16 μs | ±23.68% | 114 μs |
## Development
Before running the tests make sure you have a locally running copy of `nats-server` ([installation instructions](https://docs.nats.io/running-a-nats-service/introduction/installation)). By default, tests are run with no authentication. Make sure your NATS configuration contains no users, or has an account with [no_auth_user](https://docs.nats.io/running-a-nats-service/configuration/securing_nats/accounts#no-auth-user) explicitly enabled.
We currently use version `2.10.24` in CI, but anything higher than `2.2.0` should be fine.
Versions from `0.9.6` up to `2.2.0` should work fine for everything except header support.
Make sure to enable jetstream with the `nats-server -js` argument and you might also want to enable debug and verbose logging if you're trying to understand the messages being sent to/from nats (ie `nats-server -js -D -V`).
The typical `mix test` will run all the basic unit tests.
You can also run the `multi_server` set of tests that test connectivity to different
`nats-server` configurations. You can run these with `mix test --only multi_server` .
The tests will tell you how to start the different configurations.
There are also some property-based tests that generate a lot of test cases.
You can tune how many test cases by setting the environment variable `N=200 mix test --only property` (default it 100).
For more details you can look at how Github runs these things in the CI flow.
================================================
FILE: bench/client.exs
================================================
Application.put_env(:client, :num_connections, 4)
num_requesters = 16
requests_per_requester = 500
defmodule Client do
require Logger
def setup(id) do
num_connections = Application.get_env(:client, :num_connections)
partition = rem(id, num_connections)
String.to_atom("gnat#{partition}")
end
def send_request(gnat, request) do
{:ok, _} = Gnat.request(gnat, "echo", request)# |> IO.inspect
end
def send_requests(gnat, how_many, request) do
:lists.seq(1, how_many)
|> Enum.each(fn(_) ->
{micro_seconds, _result} = :timer.tc(fn() -> send_request(gnat, request) end)
Benchmark.record_rpc_time(micro_seconds)
end)
end
end
defmodule Benchmark do
def benchmark(num_actors, requests_per_actor, request) do
{:ok, _pid} = Agent.start_link(fn -> [] end, name: __MODULE__)
{total_micros, _result} = time_benchmark(num_actors, requests_per_actor, request)
total_requests = num_actors * requests_per_actor
total_bytes = total_requests * byte_size(request) * 2
print_statistics(total_requests, total_bytes, total_micros)
Agent.stop(__MODULE__, :normal)
end
def record_rpc_time(micro_seconds) do
Agent.update(__MODULE__, fn(list) -> [micro_seconds | list] end)
end
def print_statistics(total_requests, total_bytes, total_micros) do
total_seconds = total_micros / 1_000_000.0
req_throughput = total_requests / total_seconds
kilobyte_throughput = total_bytes / 1024 / total_seconds
IO.puts "It took #{total_seconds}sec"
IO.puts "\t#{req_throughput} req/sec"
IO.puts "\t#{kilobyte_throughput} kb/sec"
Agent.get(__MODULE__, fn(list_of_rpc_times) ->
tc_l = list_of_rpc_times
tc_n = Enum.count(list_of_rpc_times)
tc_min = :lists.min(tc_l)
tc_max = :lists.max(tc_l)
sorted = :lists.sort(tc_l)
tc_med = :lists.nth(round(tc_n * 0.5), sorted)
tc_90th = :lists.nth(round(tc_n * 0.9), sorted)
tc_avg = round(Enum.sum(tc_l) / tc_n)
IO.puts "\tmin: #{tc_min}µs"
IO.puts "\tmax: #{tc_max}µs"
IO.puts "\tmedian: #{tc_med}µs"
IO.puts "\t90th percentile: #{tc_90th}µs"
IO.puts "\taverage: #{tc_avg}µs"
IO.puts "\t#{tc_min},#{tc_max},#{tc_med},#{tc_90th},#{tc_avg},#{req_throughput},#{kilobyte_throughput}"
end)
end
def time_benchmark(num_actors, requests_per_actor, request) do
:timer.tc(fn() ->
(1..num_actors) |> Enum.map(fn(i) ->
parent = self()
spawn(fn() ->
gnat = Client.setup(i)
#IO.puts "starting requests #{i}"
Client.send_requests(gnat, requests_per_actor, request)
#IO.puts "done with requests #{i}"
send parent, :ack
end)
end)
wait_for_times(num_actors)
end)
end
def wait_for_times(0), do: :done
def wait_for_times(n) do
receive do
:ack ->
wait_for_times(n-1)
end
end
end
num_connections = Application.get_env(:client, :num_connections)
Enum.each(0..(num_connections - 1), fn(i) ->
name = :"gnat#{i}"
{:ok, _pid} = Gnat.start_link(%{}, name: name)
end)
:timer.sleep(500) # let the connections get started
#request = "ping"
request = :crypto.strong_rand_bytes(16)
Benchmark.benchmark(num_requesters, requests_per_requester, request)
================================================
FILE: bench/kv_consume.exs
================================================
# bench/kv_consume.exs
#
# Compares three approaches for consuming all messages from a KV bucket into ETS:
#
# 1. Pager (batch 500, ack_policy: :all) — fetch a page, process, ack last, repeat
# 2. Pull + ack_next pipeline (initial batch 500, ack_policy: :explicit) — prime the
# pipeline with a batch request, then ack_next each message to keep flow continuous
# 3. PullConsumer with batch_size (ack_policy: :all) — the new batch mode using the
# actual PullConsumer behaviour, batches messages and acks only the last per batch
#
# Prerequisites:
# - NATS server with JetStream enabled: nats-server -js
# - Run with: mix run bench/kv_consume.exs
#
# Optional env vars:
# - BENCH_COUNT: number of messages (default 100000)
# - BENCH_BATCH: batch size (default 500)
# - BENCH_TIME: seconds per scenario (default 60)
Logger.configure(level: :warning)
require Logger
Logger.configure(level: :warning)
alias Gnat.Jetstream.API.{Consumer, KV}
alias Gnat.Jetstream.API.Util
defmodule BenchBatchPullConsumer do
use Gnat.Jetstream.PullConsumer
def start(args) do
Gnat.Jetstream.PullConsumer.start(__MODULE__, args)
end
@impl true
def init(%{tab: tab, notify: pid, expected: expected, batch_size: batch_size}) do
consumer = %Consumer{
stream_name: "KV_BENCH_KV",
ack_policy: :all,
ack_wait: 30_000_000_000,
deliver_policy: :all,
replay_policy: :instant
}
{:ok, %{tab: tab, notify: pid, expected: expected, received: 0},
connection_name: :gnat_bench, consumer: consumer, batch_size: batch_size}
end
@impl true
def handle_message(message, state) do
:ets.insert(state.tab, {message.topic, message.body})
received = state.received + 1
if received >= state.expected do
send(state.notify, {:done, received})
end
{:ack, %{state | received: received}}
end
end
defmodule KVConsumeBench do
@bucket "BENCH_KV"
@stream "KV_BENCH_KV"
@value_size 64
def setup(conn, count) do
# Clean up previous state
KV.delete_bucket(conn, @bucket)
:timer.sleep(500)
{:ok, _} = KV.create_bucket(conn, @bucket, history: 1)
IO.puts("Populating #{count} messages (#{@value_size} byte values)...")
start = System.monotonic_time(:millisecond)
Enum.each(1..count, fn i ->
key = "key.#{String.pad_leading(Integer.to_string(i), 7, "0")}"
value = :crypto.strong_rand_bytes(@value_size) |> Base.encode64()
:ok = KV.put_value(conn, @bucket, key, value)
if rem(i, 10_000) == 0 do
elapsed = System.monotonic_time(:millisecond) - start
rate = round(i / elapsed * 1000)
IO.puts(" #{i}/#{count} (#{rate} msg/s)")
end
end)
elapsed = System.monotonic_time(:millisecond) - start
IO.puts("Setup complete: #{count} messages in #{div(elapsed, 1000)}s\n")
end
# ---------------------------------------------------------------------------
# Strategy 1: Pager (ack_policy: :all, batch fetch, ack last per page)
# ---------------------------------------------------------------------------
def pager_consume(conn, batch_size) do
tab = :ets.new(:pager_cache, [:set])
{:ok, _} =
Gnat.Jetstream.Pager.reduce(conn, @stream, [batch: batch_size], nil, fn msg, acc ->
:ets.insert(tab, {msg.topic, msg.body})
acc
end)
count = :ets.info(tab, :size)
:ets.delete(tab)
count
end
# ---------------------------------------------------------------------------
# Strategy 2: Pull + ack_next pipeline (ack_policy: :explicit, continuous)
# ---------------------------------------------------------------------------
def pull_ack_next_consume(conn, batch_size) do
{:ok, consumer_info} =
Consumer.create(conn, %Consumer{
stream_name: @stream,
ack_policy: :explicit,
deliver_policy: :all,
replay_policy: :instant,
inactive_threshold: 30_000_000_000
})
total = consumer_info.num_pending
inbox = Util.reply_inbox()
{:ok, sub} = Gnat.sub(conn, self(), inbox)
:ok =
Consumer.request_next_message(
conn,
@stream,
consumer_info.name,
inbox,
nil,
batch: batch_size,
no_wait: true
)
tab = :ets.new(:pull_cache, [:set])
receive_with_ack_next(sub, inbox, tab, 0, total)
count = :ets.info(tab, :size)
:ets.delete(tab)
Gnat.unsub(conn, sub)
Consumer.delete(conn, @stream, consumer_info.name)
count
end
@terminals ["404", "408"]
defp receive_with_ack_next(_sub, _inbox, _tab, total, total), do: :ok
defp receive_with_ack_next(sub, inbox, tab, count, total) do
receive do
{:msg, %{sid: ^sub, status: status}} when status in @terminals ->
receive_with_ack_next(sub, inbox, tab, count, total)
{:msg, %{sid: ^sub, reply_to: nil}} ->
receive_with_ack_next(sub, inbox, tab, count, total)
{:msg, %{sid: ^sub} = message} ->
:ets.insert(tab, {message.topic, message.body})
if count + 1 < total do
Gnat.Jetstream.ack_next(message, inbox)
else
Gnat.Jetstream.ack(message)
end
receive_with_ack_next(sub, inbox, tab, count + 1, total)
after
30_000 ->
IO.puts("WARNING: timeout after receiving #{count}/#{total} messages")
:timeout
end
end
# ---------------------------------------------------------------------------
# Strategy 3: PullConsumer with batch_size (ack_policy: :all, batch mode)
#
# Uses the actual PullConsumer behaviour with the new batch_size option.
# This is the real-world usage pattern we want to validate.
# ---------------------------------------------------------------------------
def batch_pull_consumer_consume(expected, batch_size) do
tab = :ets.new(:batch_pc_cache, [:set, :public])
{:ok, pid} =
BenchBatchPullConsumer.start(%{
tab: tab,
notify: self(),
expected: expected,
batch_size: batch_size
})
receive do
{:done, _received} -> :ok
after
60_000 ->
IO.puts("WARNING: PullConsumer timeout")
end
count = :ets.info(tab, :size)
Gnat.Jetstream.PullConsumer.close(pid)
:ets.delete(tab)
count
end
end
# -- Configuration -----------------------------------------------------------
count = String.to_integer(System.get_env("BENCH_COUNT", "100000"))
batch = String.to_integer(System.get_env("BENCH_BATCH", "500"))
time = String.to_integer(System.get_env("BENCH_TIME", "60"))
IO.puts("""
KV Consume Benchmark
====================
Messages: #{count}
Batch size: #{batch}
Time/scenario: #{time}s
""")
# -- Setup --------------------------------------------------------------------
# Named connection for the PullConsumer
conn_settings = %{
name: :gnat_bench,
backoff_period: 1_000,
connection_settings: [%{host: '127.0.0.1', port: 4222}]
}
{:ok, _} = Gnat.ConnectionSupervisor.start_link(conn_settings)
:timer.sleep(500)
# Direct connection for Pager and manual pull
{:ok, conn} = Gnat.start_link(%{host: '127.0.0.1', port: 4222})
KVConsumeBench.setup(conn, count)
# -- Verify all approaches produce correct results ----------------------------
IO.puts("Verifying approaches...")
count1 = KVConsumeBench.pager_consume(conn, batch)
IO.puts(" Pager: #{count1} entries")
count2 = KVConsumeBench.pull_ack_next_consume(conn, batch)
IO.puts(" Pull+ack_next: #{count2} entries")
count3 = KVConsumeBench.batch_pull_consumer_consume(count, batch)
IO.puts(" Batch PullConsumer: #{count3} entries")
expected_counts = [count1, count2, count3]
if Enum.any?(expected_counts, &(&1 != count)) do
IO.puts("\nERROR: expected #{count} entries from each approach")
Gnat.stop(conn)
System.halt(1)
end
IO.puts("\nAll approaches verified. Starting benchmark...\n")
# -- Benchmark ---------------------------------------------------------------
Benchee.run(
%{
"pager (batch #{batch})" => fn ->
KVConsumeBench.pager_consume(conn, batch)
end,
"pull+ack_next (initial batch #{batch})" => fn ->
KVConsumeBench.pull_ack_next_consume(conn, batch)
end,
"batch_pull_consumer (batch #{batch})" => fn ->
KVConsumeBench.batch_pull_consumer_consume(count, batch)
end
},
time: time,
warmup: 0,
memory_time: 0,
formatters: [{Benchee.Formatters.Console, comparisons: true}]
)
Gnat.stop(conn)
================================================
FILE: bench/parse.exs
================================================
msg1024 = :crypto.strong_rand_bytes(1024)
msg128 = :crypto.strong_rand_bytes(128)
msg16 = :crypto.strong_rand_bytes(16)
inputs = %{
"16 byte" => "MSG topic 1 16\r\n#{msg16}\r\n",
"128 byte" => "MSG topic 1 128\r\n#{msg128}\r\n",
"1024 byte" => "MSG topic 1 1024\r\n#{msg1024}\r\n",
"7 byte with headers" => "HMSG SUBJECT 1 REPLY 48 55\r\nNATS/1.0\r\nHeader1: X\r\nHeader1: Y\r\nHeader2: Z\r\n\r\nPAYLOAD\r\n"
}
parsec = Gnat.Parsec.new()
Benchee.run(%{
"parsec" => fn(tcp_packet) -> {_parse, [_msg]} = Gnat.Parsec.parse(parsec, tcp_packet) end,
}, time: 10, parallel: 1, inputs: inputs, formatters: [{Benchee.Formatters.Console, comparisons: false}])
================================================
FILE: bench/publish.exs
================================================
inputs = %{
"16 byte" => :crypto.strong_rand_bytes(16),
"128 byte" => :crypto.strong_rand_bytes(128),
"1024 byte" => :crypto.strong_rand_bytes(1024),
}
{:ok, client_pid} = Gnat.start_link(%{host: '127.0.0.1', port: 4222})
Benchee.run(%{
"pub" => fn(msg) -> :ok = Gnat.pub(client_pid, "echo", msg) end,
}, time: 10, parallel: 1, inputs: inputs, formatters: [{Benchee.Formatters.Console, comparisons: false}])
================================================
FILE: bench/request.exs
================================================
defmodule EchoServer do
def run(gnat) do
spawn(fn -> init(gnat) end)
end
def init(gnat) do
Gnat.sub(gnat, self(), "echo")
loop(gnat)
end
def loop(gnat) do
receive do
{:msg, %{topic: "echo", reply_to: reply_to, body: msg}} ->
Gnat.pub(gnat, reply_to, msg)
other ->
IO.puts "server received: #{inspect other}"
end
loop(gnat)
end
end
{:ok, server_pid} = Gnat.start_link(%{host: '127.0.0.1', port: 4222})
EchoServer.run(server_pid)
inputs = %{
"16 byte" => :crypto.strong_rand_bytes(16),
"128 byte" => :crypto.strong_rand_bytes(128),
"1024 byte" => :crypto.strong_rand_bytes(1024),
}
{:ok, client_pid} = Gnat.start_link(%{host: '127.0.0.1', port: 4222})
Benchee.run(%{
"request" => fn(msg) -> {:ok, %{body: _}} = Gnat.request(client_pid, "echo", msg) end,
}, time: 10, parallel: 1, inputs: inputs, formatters: [{Benchee.Formatters.Console, comparisons: false}])
================================================
FILE: bench/request_multi.exs
================================================
defmodule EchoServer do
def run(gnat) do
spawn(fn -> init(gnat) end)
end
def init(gnat) do
Gnat.sub(gnat, self(), "echo")
loop(gnat)
end
def loop(gnat) do
receive do
{:msg, %{topic: "echo", reply_to: reply_to, body: msg}} ->
Gnat.pub(gnat, reply_to, msg)
other ->
IO.puts "server received: #{inspect other}"
end
loop(gnat)
end
end
{:ok, server_pid} = Gnat.start_link(%{host: '127.0.0.1', port: 4222})
# run 3 servers to get 3 responses
EchoServer.run(server_pid)
EchoServer.run(server_pid)
EchoServer.run(server_pid)
inputs = %{
"16 byte" => :crypto.strong_rand_bytes(16),
"128 byte" => :crypto.strong_rand_bytes(128),
"1024 byte" => :crypto.strong_rand_bytes(1024),
}
{:ok, client_pid} = Gnat.start_link(%{host: '127.0.0.1', port: 4222})
Benchee.run(%{
"request_multi" => fn(msg) -> {:ok, [%{body: _}, %{}, %{}]} = Gnat.request_multi(client_pid, "echo", msg, max_messages: 3) end,
}, time: 10, parallel: 1, inputs: inputs, formatters: [{Benchee.Formatters.Console, comparisons: false}])
================================================
FILE: bench/server.exs
================================================
num_connections = 4
num_subscribers = 4
Enum.each(0..(num_connections - 1), fn(i) ->
name = :"gnat#{i}"
{:ok, _pid} = Gnat.start_link(%{}, name: name)
end)
Enum.each(0..(num_subscribers - 1), fn(i) ->
name = :"consumer#{i}"
conn_name = :"gnat#{rem(i, num_connections)}"
IO.puts "#{name} will use #{conn_name}"
{:ok, _pid} = Gnat.ConsumerSupervisor.start_link(%{connection_name: conn_name, consuming_function: {EchoServer, :handle}, subscription_topics: [%{topic: "echo", queue_group: "echo"}]})
end)
defmodule EchoServer do
def handle(%{body: body, reply_to: reply_to, gnat: gnat_pid}) do
Gnat.pub(gnat_pid, reply_to, body)
end
def wait_loop do
:timer.sleep(1_000)
wait_loop()
end
end
EchoServer.wait_loop()
================================================
FILE: bench/service_bench.exs
================================================
defmodule EchoService do
use Gnat.Services.Server
def request(%{body: body}, "echo", _group) do
{:reply, body}
end
def definition do
%{
name: "echo",
description: "This is an example service",
version: "0.0.1",
endpoints: [
%{
name: "echo",
group_name: "mygroup",
}
]
}
end
end
conn_supervisor_settings = %{
name: :gnat, # (required) the registered named you want to give the Gnat connection
backoff_period: 1_000, # number of milliseconds to wait between consecutive reconnect attempts (default: 2_000)
connection_settings: [
%{host: '127.0.0.1', port: 4222},
]
}
{:ok, _pid} = Gnat.ConnectionSupervisor.start_link(conn_supervisor_settings)
# let the connection get established
:timer.sleep(100)
consumer_supervisor_settings = %{
connection_name: :gnat,
module: EchoService, # a module that implements the Gnat.Services.Server behaviour
service_definition: EchoService.definition()
}
{:ok, _pid} = Gnat.ConsumerSupervisor.start_link(consumer_supervisor_settings)
# wait for the connection and consumer to be ready
:timer.sleep(2000)
{:ok, client_pid} = Gnat.start_link(%{host: '127.0.0.1', port: 4222})
inputs = %{
"16 byte" => :crypto.strong_rand_bytes(16),
"256 byte" => :crypto.strong_rand_bytes(256),
"1024 byte" => :crypto.strong_rand_bytes(1024),
}
Benchee.run(%{
"service" => fn(msg) -> {:ok, %{body: ^msg}} = Gnat.request(client_pid, "mygroup.echo", msg) end,
}, time: 10, parallel: 1, inputs: inputs, formatters: [{Benchee.Formatters.Console, comparisons: false}])
================================================
FILE: dependencies.md
================================================
# Project Dependencies
This is a list of dependencies that will be pulled into your project when you use this library. Unless otherwise specified all dependencies are Hex packages obtained via mix.
This list of dependencies was produced on October 13 2023.
| Dependency | License |
|-|-|
| ed25519 1.4.0 | MIT |
| cowlib 2.11.0 | ISC |
| deep_merge 1.0.0 | MIT |
| jason 1.2.2 | Apache 2.0 |
| nimble_parsec 1.2.0 | Apache 2.0 |
| nkeys 0.2.1 | MIT |
| telemetry 1.0.0 (rebar3) | Apache 2.0 |
# Development Dependencies
This is a list of dependencies used to build and test this library.
| Dependency | License |
|-|-|
| propcheck 1.4.1 | GPL 3.0 |
| proper 1.4.0 | GPL 3.0 |
| erlex 0.2.6 | Apache 2.0 |
| makeup 1.0.5 | BSD |
| makeup_elixir 0.15.2 | BSD |
| makeup_erlang 0.1.1 | BSD |
| deep_merge 0.2.0 | MIT |
| benchee 1.0.1 | MIT |
| dialyxir 1.1.0 | Apache 2.0 |
| earmark 1.3.1 | Apache 2.0 |
| makeup_elixir 0.13.0 | BSD |
| earmark_parser 1.4.18 | Apache-2.0 |
| ex_doc 0.26.0 | Apache 2.0 |
================================================
FILE: docs/js/guides/broadway.md
================================================
# Using Broadway with Jetstream
Broadway is a library which allows building concurrent and multi-stage data ingestion and data
processing pipelines with Elixir easily. You can learn about it more in
[Broadway documentation](https://hexdocs.pm/broadway/introduction.html).
Jetstream library comes with tools necessary to use NATS Jetstream with Broadway.
## Getting started
In order to use Broadway with NATS Jetstream you need to:
1. Setup a NATS Server with JetStream turned on
2. Create stream and consumer on NATS server
3. Configure Gnat connection in your Elixir project
4. Configure your project to use Broadway
In this guide, we are going to focus on the fourth point. To learn how to start Jetstream locally
with Docker Compose and then add Gnat and Jetstream to your application, see the Starting Jetstream
section in [Getting Started guide](../introduction/getting_started.md).
### Adding Broadway to your application
Once we have NATS with JetStream running and the stream and consumer we are going to use are
created, we can proceed to adding Broadway to our project. First, put `:broadway` to the list of
dependencies in `mix.exs`.
```elixir
defp deps do
[
...
{:broadway, ...version...},
...
]
end
```
Visit [Broadway page on Hex.pm](https://hex.pm/packages/broadway) to check for current version
to put in `deps`.
To install the dependencies, run:
```shell
mix deps.get
```
### Defining the pipeline configuration
The next step is to define your Broadway module. We need to implement three functions in order
to define a Broadway pipeline: `start_link/1`, `handle_message/3` and `handle_batch/4`.
Let's create `start_link/1` first:
```elixir
defmodule MyBroadway do
use Broadway
alias Broadway.Message
def start_link(_opts) do
Broadway.start_link(
__MODULE__,
name: MyBroadwayExample,
producer: [
module: {
OffBroadway.Jetstream.Producer,
connection_name: :gnat,
stream_name: "TEST_STREAM",
consumer_name: "TEST_CONSUMER"
},
concurrency: 10
],
processors: [
default: [concurrency: 10]
],
batchers: [
default: [
concurrency: 5,
batch_size: 10,
batch_timeout: 2_000
]
]
...
)
end
...callbacks..
end
```
All `start_link/1` does is just delegating to `Broadway.start_link/2`.
To understand what all these options mean and to learn about other possible settings, visit
[Broadway documentation](https://hexdocs.pm/broadway/Broadway.html).
The part that interests us the most in this guide is the `producer.module`. Here we're choosing
`OffBroadway.Jetstream.Producer` as the producer module and passing the connection options,
such as Gnat process name and stream name. For full list of available options, visit
[Producer](`OffBroadway.Jetstream.Producer`) documentation.
### Implementing Broadway callbacks
Broadway requires some callbacks to be implemented in order to process messages. For full list
of available callbacks visit
[Broadway documentation](https://hexdocs.pm/broadway/Broadway.html#callbacks).
A simple example:
```elixir
defmodule MyBroadway do
use Broadway
alias Broadway.Message
...start_link...
def handle_message(_processor_name, message, _context) do
message
|> Message.update_data(&process_data/1)
|> case do
"FOO" -> Message.configure_ack(on_success: :term)
"BAR" -> Message.configure_ack(on_success: :nack)
message -> message
end
end
defp process_data(data) do
String.upcase(data)
end
def handle_batch(_, messages, _, _) do
list = messages |> Enum.map(fn e -> e.data end)
IO.puts("Got a batch: #{inspect(list)}. Sending acknowledgements...")
messages
end
```
First, in `handle_message/3` we update our messages' data individually by converting them to
uppercase. Then, in the same callback, we're changing the success ack option of the message
to `:term` if its content is `"FOO"` or to `:nack` if the message is `"BAR"`. In the end we
print each batch in `handle_batch/4`. It's not quite useful but should be enough for this
guide.
## Running the Broadway pipeline
Once we have our pipeline fully defined, we need to add it as a child in the supervision tree.
Most applications have a supervision tree defined at `lib/my_app/application.ex`.
```elixir
children = [
{MyBroadway, []}
]
Supervisor.start_link(children, strategy: :one_for_one)
```
You can now test the pipeline. Let's start the application:
```shell
iex -S mix
```
Use Gnat API to send messages to your stream:
```elixir
Gnat.pub(:gnat, "test_subject", "foo")
Gnat.pub(:gnat, "test_subject", "bar")
Gnat.pub(:gnat, "test_subject", "baz")
```
Batcher should then print:
```
Got a batch: ["FOO", "BAR", "BAZ"]. Sending acknowledgements...
```
================================================
FILE: docs/js/guides/managing.md
================================================
# Managing Streams and Consumers
Jetstream provides a JSON API for managing streams and consumers.
This library exposes this API via interactions with the `Jetstream.Api.Stream` and `Jetstream.Api.Consumer` modules.
These modules act as native wrappers for the API and do not attempt to simplify any of the common use-cases.
As this library matures we may introduce a separate layer of functions to handle these scenarios, but for now our aim is to provide full access to the Jetstream API.
================================================
FILE: docs/js/guides/push_based_consumer.md
================================================
# Push based consumer
```elixir
# Start a nats server with jetstream enabled and default configs
# Now run the following snippets in an IEx terminal
alias Jetstream.API.{Consumer,Stream}
# Setup a connection to the nats server and create the stream/consumer
# This is the equivalent of these two nats cli commands
# nats stream add TEST --subjects="greetings" --max-msgs=-1 --max-msg-size=-1 --max-bytes=-1 --max-age=-1 --storage=file --retention=limits --discard=old
# nats consumer add TEST TEST --target consumer.greetings --replay instant --deliver=all --ack all --wait=5s --filter="" --max-deliver=10
{:ok, connection} = Gnat.start_link()
stream = %Stream{name: "TEST", subjects: ["greetings"]}
{:ok, _response} = Stream.create(connection, stream)
consumer = %Consumer{stream_name: "TEST", name: "TEST", deliver_subject: "consumer.greetings", ack_wait: 5_000_000_000, max_deliver: 10}
{:ok, _response} = Consumer.create(connection, consumer)
# Setup Consuming Function
defmodule Subscriber do
def handle(msg) do
IO.inspect(msg)
case msg.body do
"hola" -> Jetstream.ack(msg)
"bom dia" -> Jetstream.nack(msg)
_ -> nil
end
end
end
# normally you would add the `ConnectionSupervisor` and `ConsumerSupervisor` to your supervisrion tree
# here we start them up manually in an IEx session
{:ok, _pid} = Gnat.ConnectionSupervisor.start_link(%{
name: :gnat,
backoff_period: 4_000,
connection_settings: [
%{}
]
})
{:ok, _pid} = Gnat.ConsumerSupervisor.start_link(%{
connection_name: :gnat,
consuming_function: {Subscriber, :handle},
subscription_topics: [
%{topic: "consumer.greetings"}
]
})
# now publish some messages into the stream
Gnat.pub(:gnat, "greetings", "hello") # no ack will be sent back, so you'll see this message received 10 times with a 5sec pause between each one
Gnat.pub(:gnat, "greetings", "hola") # an ack is sent back so this will only be received once
Gnat.pub(:gnat, "greetings", "bom dia") # a -NAK is sent back so you'll see this received 10 times very quickly
```
================================================
FILE: docs/js/introduction/getting_started.md
================================================
# Getting Started
In this guide, we're going to learn how to install Jetstream in your project and start consuming
messages from your streams.
## Starting Jetstream
The following Docker Compose file will do the job:
```yaml
version: "3"
services:
nats:
image: nats:latest
command:
- -js
ports:
- 4222:4222
```
Save this snippet as `docker-compose.yml` and run the following command:
```shell
docker compose up -d
```
Let's also create Jetstream stream where we will publish our hello world messages:
```shell
nats stream add HELLO --subjects="greetings"
```
> #### Tip {: .tip}
>
> You can also manage Jetstream streams and consumers via Elixir. You can see more details in
> [this guide](../guides/managing.md).
## Adding Jetstream and Gnat to an application
To start off with, we'll generate a new Elixir application by running this command:
```
mix new hello_jetstream --sup
```
We need to have [a supervision tree](http://elixir-lang.org/getting-started/mix-otp/supervisor-and-application.html)
up and running in your app, and the `--sup` option ensures that.
To add Jetstream to this application, you need to add [Jetstream](https://hex.pm/packages/jetstream)
and [Gnat](https://hex.pm/packages/gnat) libraries to your `deps` definition in our `mix.exs` file.
**Fill exact version requirements from each package Hex.pm pages.**
```elixir
defp deps do
[
{:gnat, ...},
{:jetstream, ...}
]
end
```
To install these dependencies, we will run this command:
```shell
mix deps.get
```
Now let's connect to our NATS server. To do this, you need to start `Gnat.ConnectionSupervisor`
under our application's supervision tree. Add following to `lib/hello_jetstream/application.ex`:
```elixir
def start(_type, _args) do
children = [
...
# Create NATS connection
{Gnat.ConnectionSupervisor,
%{
name: :gnat,
connection_settings: [
%{host: "localhost", port: 4222}
]
}},
]
...
```
This piece of configuration will start Gnat processes that connect to the NATS server and allow
publishing and subscribing to any subjects. Jetstream operates using plain NATS subjects which
follow specific naming and message format conventions.
Let's now create a _pull consumer_ which will subscribe a specific Jetstream stream and print
incoming messages to standard output.
## Creating a pull consumer
Jetstream requires us to allocate a view/cursor of the stream that our consumer will operate on.
In Jetstream terminology, this view is called a _consumer_ (Funnily enough we've just implemented
a consumer in our code, coincidence?). [Jetstream](https://docs.nats.io/nats-concepts/jetstream/consumers)
[documentation](https://docs.nats.io/nats-concepts/jetstream/consumers/example_configuration)
offers great insights on benefits of having this separate concept so we won't duplicate work here.
Jetstream offers two stream consuming modes: _push_ and _pull_.
In _push_ mode, Jetstream will simply send messages to selected consumers immediately when they are
received. This approach does offer congestion control, so it is not recommended for high-volume
and/or reliability sensitive streams. You do not really need this library to implement push
consumer because all building blocks are in `Gnat` library. You can read more about push consumers
in [this guide](../guides/push_based_consumer.md).
On the other hand, in _pull_ mode consumers ask Jetstream for more messages when they are ready
to process them. This is the recommended approach for most use cases and we will proceed with it
in this guide.
> #### This is just a brief outline {: .tip}
>
> For more details about differences between consumer modes, consult
> [Jetstream documentation](https://docs.nats.io/nats-concepts/jetstream/consumers).
Let's create a pull consumer module within our application at
`lib/hello_jetstream/logger_pull_consumer.ex`:
```elixir
defmodule HelloJetstream.LoggerPullConsumer do
use Jetstream.PullConsumer
def start_link([]) do
Jetstream.PullConsumer.start_link(__MODULE__, [])
end
@impl true
def init([]) do
{:ok, nil, connection_name: :gnat, stream_name: "HELLO", consumer_name: "LOGGER"}
end
@impl true
def handle_message(message, state) do
IO.inspect(message)
{:ack, state}
end
end
```
Pull Consumer is a regular `GenServer` and it takes a reference to `Gnat.ConnectionSupervisor`
along with names of Jetstream stream and consumer as options passed to
`Jetstream.PullConsumer.start*` functions. These options are passed as keyword list in third element
of tuple returned from the `c:Jetstream.PullConsumer.init/1` callback.
The only required callbacks are well known gen server's `c:Jetstream.PullConsumer.init/1` and
`c:Jetstream.PullConsumer.handle_message/2`, which takes new message as its first argument and
is expected to return an _ACK action_ instructing underlying process loop what to do with this
message. Here we are asking it to automatically send for us an ACK message back to Jetstream.
Let's now create a consumer in our NATS server. We will call it `LOGGER` as we plan to let it simply
log everything published to the stream.
```shell
nats consumer add --pull --deliver=all HELLO LOGGER
```
Now, let's start our pull consumer under application's supervision tree.
```elixir
def start(_type, _args) do
children = [
...
# Jetstream Pull Consumer
HelloJetstream.LoggerPullConsumer,
]
...
```
Let's now publish some messages to our `HELLO` stream, so something will be waiting for our
application to be read when it starts.
## Publishing messages to streams
Jetstream listens on regular NATS subjects, so publishing messages is dead simple with `Gnat.pub/3`:
```elixir
Gnat.pub(:gnat, "greetings", "Hello World")
```
Or via NATS CLI:
```shell
nats pub greetings "Hello World"
```
That's it! When you run your app, you should see your messages being read by your application.
================================================
FILE: docs/js/introduction/overview.md
================================================
# Overview
[Jetstream](https://docs.nats.io/nats-concepts/jetstream) is a distributed persistence system
built-in to [NATS](https://nats.io/). It provides a streaming system that lets you capture streams
of events from various sources and persist these into persistent stores, which you can immediately
or later replay for processing.
This library exposes interfaces for publishing, consuming and managing Jetstream services. It builds
on top of [Gnat](https://hex.pm/packages/gnat), the officially supported Elixir client for NATS.
* [Let's get Jetstream up and running](./getting_started.md)
* [Using Broadway with Jetstream](../guides/broadway.md)
* [Pull Consumer API](`Gnat.Jetstream.PullConsumer`)
* [Create, update and delete Jetstream streams and consumers via Elixir](../guides/managing.md)
================================================
FILE: lib/gnat/command.ex
================================================
defmodule Gnat.Command do
@moduledoc false
@newline "\r\n"
@hpub "HPUB"
@pub "PUB"
@sub "SUB"
@unsub "UNSUB"
def build(:pub, topic, payload, []),
do: [@pub, " ", topic, " #{IO.iodata_length(payload)}", @newline, payload, @newline]
def build(:pub, topic, payload, reply_to: reply),
do: [
@pub,
" ",
topic,
" ",
reply,
" #{IO.iodata_length(payload)}",
@newline,
payload,
@newline
]
def build(:pub, topic, payload, headers: headers) do
# it takes 10 bytes to add the nats header version line
# and 2 more for the newline between headers and payload
header_len = IO.iodata_length(headers) + 12
total_len = IO.iodata_length(payload) + header_len
[
@hpub,
" ",
topic,
" ",
Integer.to_string(header_len),
" ",
Integer.to_string(total_len),
"\r\nNATS/1.0\r\n",
headers,
@newline,
payload,
@newline
]
end
def build(:pub, topic, payload, headers: headers, reply_to: reply) do
# it takes 10 bytes to add the nats header version line
# and 2 more for the newline between headers and payload
header_len = IO.iodata_length(headers) + 12
total_len = IO.iodata_length(payload) + header_len
[
@hpub,
" ",
topic,
" ",
reply,
" ",
Integer.to_string(header_len),
" ",
Integer.to_string(total_len),
"\r\nNATS/1.0\r\n",
headers,
@newline,
payload,
@newline
]
end
def build(:sub, topic, sid, []), do: [@sub, " ", topic, " ", Integer.to_string(sid), @newline]
def build(:sub, topic, sid, queue_group: qg),
do: [@sub, " ", topic, " ", qg, " ", Integer.to_string(sid), @newline]
def build(:unsub, sid, []), do: [@unsub, " #{sid}", @newline]
def build(:unsub, sid, max_messages: max), do: [@unsub, " #{sid}", " #{max}", @newline]
end
================================================
FILE: lib/gnat/connection_supervisor.ex
================================================
defmodule Gnat.ConnectionSupervisor do
use GenServer
require Logger
@moduledoc """
A process that can supervise a named connection for you
If you would like to supervise a Gnat connection and have it automatically re-connect in case of failure you can use this module in your supervision tree.
It takes a map with the following data:
```
gnat_supervisor_settings = %{
name: :gnat, # (required) the registered named you want to give the Gnat connection
backoff_period: 4_000, # number of milliseconds to wait between consecutive reconnect attempts (default: 2_000)
connection_settings: [
%{host: '10.0.0.100', port: 4222},
%{host: '10.0.0.101', port: 4222},
]
}
```
The connection settings can specify all of the same values that you pass to `Gnat.start_link/1`. Each time a connection is attempted we will use one of the provided connection settings to open the connection. This is a simplistic way of load balancing your connections across a cluster of nats nodes and allowing failover to other nodes in the cluster if one goes down.
To use this in your supervision tree add an entry like this:
```
import Supervisor.Spec
worker(Gnat.ConnectionSupervisor, [gnat_supervisor_settings, [name: :my_connection_supervisor]])
```
The second argument is used as GenServer options so you can give the supervisor a registered name as well if you like. Now in the rest of your code you can call things like:
```
:ok = Gnat.pub(:gnat, "subject", "message")
```
And it will use your supervised connection. If the connection is down when you call that function (or dies during that function) it will raise an error.
"""
@spec start_link(map(), keyword()) :: GenServer.on_start()
def start_link(settings, options \\ []) do
GenServer.start_link(__MODULE__, settings, options)
end
@impl GenServer
def init(options) do
state = %{
backoff_period: Map.get(options, :backoff_period, 2000),
connection_settings: Map.fetch!(options, :connection_settings),
name: Map.fetch!(options, :name),
gnat: nil
}
Process.flag(:trap_exit, true)
send(self(), :attempt_connection)
{:ok, state}
end
@impl GenServer
def handle_info(:attempt_connection, state) do
connection_config = random_connection_config(state)
Logger.debug("connecting to #{inspect(connection_config)}")
case Gnat.start_link(connection_config, name: state.name) do
{:ok, gnat} ->
{:noreply, %{state | gnat: gnat}}
{:error, err} ->
Logger.error("failed to connect #{inspect(err)}")
Process.send_after(self(), :attempt_connection, state.backoff_period)
{:noreply, %{state | gnat: nil}}
end
end
# in OTP 25 and below, we will get back an EXIT message in addition to receiving the {:error, reason}
# tuple on from the start_link call above. So if we get an exit message when there is no connection tracked
# it means will have already scheduled a new attempt_connection
def handle_info({:EXIT, _pid, _reason}, %{gnat: nil} = state) do
{:noreply, state}
end
def handle_info({:EXIT, _pid, reason}, state) do
Logger.error("connection failed #{inspect(reason)}")
send(self(), :attempt_connection)
{:noreply, state}
end
def handle_info(msg, state) do
Logger.error("#{__MODULE__} received unexpected message #{inspect(msg)}")
{:noreply, state}
end
defp random_connection_config(%{connection_settings: connection_settings}) do
connection_settings |> Enum.random()
end
end
================================================
FILE: lib/gnat/consumer_supervisor.ex
================================================
defmodule Gnat.ConsumerSupervisor do
use GenServer
require Logger
alias Gnat.Services.Service
@moduledoc """
A process that can supervise consumers for you
If you want to subscribe to a few topics and have that subscription last across restarts for you, then this worker can be of help.
It also spawns a supervised `Task` for each message it receives.
This way errors in message processing don't crash the consumers, but you will still get SASL reports that you can send to services like honeybadger.
To use this just add an entry to your supervision tree like this:
```
consumer_supervisor_settings = %{
connection_name: :name_of_supervised_connection,
module: MyApp.Server, # a module that implements the Gnat.Server behaviour
subscription_topics: [
%{topic: "rpc.MyApp.search", queue_group: "rpc.MyApp.search"},
%{topic: "rpc.MyApp.create", queue_group: "rpc.MyApp.create"},
],
}
worker(Gnat.ConsumerSupervisor, [consumer_supervisor_settings, [name: :rpc_consumer]], shutdown: 30_000)
```
The second argument is a keyword list that gets used as the GenServer options so you can pass a name that you want to register for the consumer process if you like. The `:consuming_function` specifies which module and function to call when messages arrive. The function will be called with a single argument which is a `t:Gnat.message/0` just like you get when you call `Gnat.sub/4` directly.
You can have a single consumer that subscribes to multiple topics or multiple consumers that subscribe to different topics and call different consuming functions. It is recommended that your `ConsumerSupervisor`s are present later in your supervision tree than your `ConnectionSupervisor`. That way during a shutdown the `ConsumerSupervisor` can attempt a graceful shutdown of the consumer before shutting down the connection.
If you want this consumer supervisor to host a NATS service, then you can specify a module that
implements the `Gnat.Services.Server` behavior. You'll need to specify the `service_definition` field in the consumer
supervisor settings and conforms to the `Gnat.Services.Server.service_configuration` type. Here is an example of configuring
the consumer supervisor to manage a service:
```
consumer_supervisor_settings = %{
connection_name: :name_of_supervised_connection,
module: MyApp.Service, # a module that implements the Gnat.Services.Server behaviour
service_definition: %{
name: "exampleservice",
description: "This is an example service",
version: "0.1.0",
endpoints: [
%{
name: "add",
group_name: "calc",
},
%{
name: "sub",
group_name: "calc"
}
]
}
}
worker(Gnat.ConsumerSupervisor, [consumer_supervisor_settings, [name: :myservice_consumer]], shutdown: 30_000)
```
It's also possible to pass a `%{consuming_function: {YourModule, :your_function}}` rather than a `:module` in your settings.
In that case no error handling or replying is taking care of for you, microservices cannot be used, and it will be up to your function to take whatever action you want with each message.
"""
@spec start_link(map(), keyword()) :: GenServer.on_start()
def start_link(settings, options \\ []) do
GenServer.start_link(__MODULE__, settings, options)
end
@impl GenServer
def init(settings) do
Process.flag(:trap_exit, true)
{:ok, task_supervisor_pid} = Task.Supervisor.start_link()
connection_name = Map.get(settings, :connection_name)
subscription_topics = Map.get(settings, :subscription_topics)
state = %{
connection_name: connection_name,
connection_pid: nil,
svc_responder_pid: nil,
status: :disconnected,
subscription_topics: subscription_topics,
subscriptions: [],
task_supervisor_pid: task_supervisor_pid
}
with {:ok, state} <- maybe_append_service(state, settings),
{:ok, state} <- maybe_append_module(state, settings),
{:ok, state} <- maybe_append_consuming_function(state, settings),
:ok <- validate_state(state) do
send(self(), :connect)
{:ok, state}
end
end
@impl GenServer
def handle_info(:connect, %{connection_name: name} = state) do
case Process.whereis(name) do
nil ->
Process.send_after(self(), :connect, 2_000)
{:noreply, state}
connection_pid ->
_ref = Process.monitor(connection_pid)
subscriptions = subscribe_to_topics(state, connection_pid)
{:noreply,
%{
state
| status: :connected,
connection_pid: connection_pid,
subscriptions: subscriptions
}}
end
end
def handle_info(
{:DOWN, _ref, :process, connection_pid, _reason},
%{connection_pid: connection_pid} = state
) do
Process.send_after(self(), :connect, 2_000)
{:noreply, %{state | status: :disconnected, connection_pid: nil, subscriptions: []}}
end
# Ignore DOWN and task result messages from the spawned tasks
def handle_info({:DOWN, _ref, :process, _task_pid, _reason}, state), do: {:noreply, state}
def handle_info({ref, _result}, state) when is_reference(ref), do: {:noreply, state}
def handle_info(
{:EXIT, supervisor_pid, _reason},
%{task_supervisor_pid: supervisor_pid} = state
) do
{:ok, task_supervisor_pid} = Task.Supervisor.start_link()
{:noreply, Map.put(state, :task_supervisor_pid, task_supervisor_pid)}
end
def handle_info({:msg, gnat_message}, %{service: service, module: module} = state) do
Task.Supervisor.async_nolink(state.task_supervisor_pid, Gnat.Services.Server, :execute, [
module,
gnat_message,
service
])
{:noreply, state}
end
def handle_info({:msg, gnat_message}, %{module: module} = state) do
Task.Supervisor.async_nolink(state.task_supervisor_pid, Gnat.Server, :execute, [
module,
gnat_message
])
{:noreply, state}
end
def handle_info({:msg, gnat_message}, %{consuming_function: {mod, fun}} = state) do
Task.Supervisor.async_nolink(state.task_supervisor_pid, mod, fun, [gnat_message])
{:noreply, state}
end
def handle_info(other, state) do
Logger.error("#{__MODULE__} received unexpected message #{inspect(other)}")
{:noreply, state}
end
@impl GenServer
def terminate(:shutdown, state) do
Logger.info("#{__MODULE__} starting graceful shutdown")
Enum.each(state.subscriptions, fn subscription ->
:ok = Gnat.unsub(state.connection_pid, subscription)
end)
# wait for final messages from broker
Process.sleep(500)
receive_final_broker_messages(state)
wait_for_empty_task_supervisor(state)
Logger.info("#{__MODULE__} finished graceful shutdown")
end
def terminate(reason, _state) do
Logger.error("#{__MODULE__} unexpected shutdown #{inspect(reason)}")
end
defp receive_final_broker_messages(state) do
receive do
info ->
handle_info(info, state)
receive_final_broker_messages(state)
after
0 ->
:done
end
end
defp wait_for_empty_task_supervisor(%{task_supervisor_pid: pid} = state) do
case Task.Supervisor.children(pid) do
[] ->
:ok
children ->
Logger.info("#{__MODULE__}\t\t#{Enum.count(children)} tasks remaining")
Process.sleep(1_000)
wait_for_empty_task_supervisor(state)
end
end
defp subscribe_to_topics(%{service: service}, connection_pid) do
Service.subscription_topics_with_queue_group(service)
|> Enum.map(fn
{topic, nil} ->
{:ok, subscription} = Gnat.sub(connection_pid, self(), topic)
subscription
{topic, queue_group} ->
{:ok, subscription} = Gnat.sub(connection_pid, self(), topic, queue_group: queue_group)
subscription
end)
end
defp subscribe_to_topics(state, connection_pid) do
Enum.map(state.subscription_topics, fn topic_and_queue_group ->
topic = Map.fetch!(topic_and_queue_group, :topic)
{:ok, subscription} =
case Map.get(topic_and_queue_group, :queue_group) do
nil -> Gnat.sub(connection_pid, self(), topic)
queue_group -> Gnat.sub(connection_pid, self(), topic, queue_group: queue_group)
end
subscription
end)
end
defp maybe_append_service(state, %{service_definition: config}) do
case Service.init(config) do
{:ok, service} ->
{:ok, Map.put(state, :service, service)}
{:error, errors} ->
{:stop, "Invalid service configuration: #{Enum.join(errors, ",")}"}
end
end
defp maybe_append_service(state, _), do: {:ok, state}
defp maybe_append_module(state, %{module: module}) do
{:ok, Map.put(state, :module, module)}
end
defp maybe_append_module(state, _), do: {:ok, state}
defp maybe_append_consuming_function(state, %{consuming_function: consuming_function}) do
{:ok, Map.put(state, :consuming_function, consuming_function)}
end
defp maybe_append_consuming_function(state, _), do: {:ok, state}
defp validate_state(state) do
partial = Map.take(state, [:module, :consuming_function])
case Enum.count(partial) do
0 ->
{:stop, "You must provide a module or consuming function for the consumer supervisor"}
1 ->
:ok
_ ->
{:stop,
"You cannot provide both a module and consuming function. Please specify one or the other."}
end
end
end
================================================
FILE: lib/gnat/handshake.ex
================================================
defmodule Gnat.Handshake do
@moduledoc false
alias Gnat.Parsec
@doc """
This function handles all of the variations of establishing a connection to
a nats server and just returns {:ok, socket} or {:error, reason}
"""
def connect(settings) do
host = settings.host |> to_charlist
case :gen_tcp.connect(host, settings.port, settings.tcp_opts, settings.connection_timeout) do
{:ok, tcp} -> perform_handshake(tcp, settings)
result -> result
end
end
def negotiate_settings(server_settings, user_settings) do
auth_required = server_settings[:auth_required] || user_settings[:auth_required] || false
%{verbose: false}
|> negotiate_auth(server_settings, user_settings, auth_required)
|> negotiate_headers(server_settings, user_settings)
|> negotiate_no_responders(server_settings, user_settings)
end
defp perform_handshake(tcp, user_settings) do
receive do
{:tcp, ^tcp, operation} ->
{_, [{:info, server_settings}]} = Parsec.parse(Parsec.new(), operation)
{:ok, socket} = upgrade_connection(tcp, user_settings)
settings = negotiate_settings(server_settings, user_settings)
:ok = send_connect(user_settings, settings, socket)
{:ok, socket, server_settings}
after
1000 ->
{:error, "timed out waiting for info"}
end
end
defp send_connect(%{tls: true}, settings, socket) do
:ssl.send(socket, "CONNECT " <> Jason.encode!(settings, maps: :strict) <> "\r\n")
end
defp send_connect(_, settings, socket) do
:gen_tcp.send(socket, "CONNECT " <> Jason.encode!(settings, maps: :strict) <> "\r\n")
end
defp negotiate_auth(
settings,
_server,
%{username: username, password: password} = _user,
true = _auth_required
) do
Map.merge(settings, %{user: username, pass: password})
end
defp negotiate_auth(settings, _server, %{token: token} = _user, true = _auth_required) do
Map.merge(settings, %{auth_token: token})
end
defp negotiate_auth(
settings,
%{nonce: nonce} = _server,
%{nkey_seed: seed, jwt: jwt} = _user,
true = _auth_required
) do
{:ok, nkey} = NKEYS.from_seed(seed)
signature = NKEYS.sign(nkey, nonce) |> Base.url_encode64() |> String.replace("=", "")
Map.merge(settings, %{sig: signature, protocol: 1, jwt: jwt})
end
defp negotiate_auth(
settings,
%{nonce: nonce} = _server,
%{nkey_seed: seed} = _user,
true = _auth_required
) do
{:ok, nkey} = NKEYS.from_seed(seed)
signature = NKEYS.sign(nkey, nonce) |> Base.url_encode64() |> String.replace("=", "")
public = NKEYS.public_nkey(nkey)
Map.merge(settings, %{sig: signature, protocol: 1, nkey: public})
end
defp negotiate_auth(settings, _server, _user, _auth_required) do
settings
end
defp negotiate_headers(settings, %{headers: true} = _server, user_settings) do
if Map.get(user_settings, :headers, true) do
Map.put(settings, :headers, true)
else
Map.put(settings, :headers, false)
end
end
defp negotiate_headers(_settings, _server, %{headers: true} = _user) do
raise "NATS Server does not support headers, but your connection settings specify header support"
end
defp negotiate_headers(settings, _server, _user) do
settings
end
defp negotiate_no_responders(%{headers: true} = settings, _server_settings, %{
no_responders: true
}) do
Map.put(settings, :no_responders, true)
end
defp negotiate_no_responders(settings, _server_settings, _user_settings) do
settings
end
defp upgrade_connection(tcp, %{tls: true, ssl_opts: opts}) do
:ok = :inet.setopts(tcp, active: true)
:ssl.connect(tcp, opts, 1_000)
end
defp upgrade_connection(tcp, _settings), do: {:ok, tcp}
end
================================================
FILE: lib/gnat/jetstream/api/consumer.ex
================================================
defmodule Gnat.Jetstream.API.Consumer do
@moduledoc """
A module representing a NATS JetStream Consumer.
Learn more about consumers: https://docs.nats.io/nats-concepts/jetstream/consumers
## The Jetstream.API.Consumer struct
The struct's only mandatory field to set is the `:stream_name`. The rest will have
the NATS default values set.
Note that consumers are ephemeral by default. Set the `:durable_name` to make it durable.
Consumer struct fields explanation:
* `:stream_name` - name of a stream the consumer is pointing at.
* `:domain` - JetStream domain the stream is on.
* `:ack_policy` - how the messages should be acknowledged. It has the following options:
- `:explicit` - the default policy. It means that each individual message must be acknowledged.
It is the only allowed option for pull consumers.
- `:none` - no need to ack messages, the server will assume ack on delivery.
- `:all` - only the last received message needs to be acked, all the previous messages received
are automatically acknowledged.
* `:ack_wait` - time in nanoseconds that server will wait for an ack for any individual. If an ack
is not received in time, the message will be redelivered.
* `:backoff` - list of durations that represents a retry timescale for NAK'd messages or those being
normally retried.
* `:deliver_group` - when set, will only deliver messages to subscriptions matching that group.
* `:deliver_policy` - specifies where in the stream it wants to start receiving messages. It has the
following options:
- `:all` - the default policy. The consumer will start receiving from the earliest available
message.
- `:last` - the consumer will start receiving messages with the last message added to the stream.
- `:new` - the consumer will only start receiving messages that were created after the customer
was created.
- `:by_start_sequence` - the consumer is required to specify `:opt_start_seq`, the sequence number
to start on. It will receive the closest available message moving forward in the sequence
should the message specified have been removed based on the stream limit policy.
- `:by_start_time` - the consumer will start with messages on or after this time. The consumer is
required to specify `:opt_start_time`, the time in the stream to start at.
- `:last_per_subject` - the consumer will start with the latest one for each filtered subject
currently in the stream.
* `:deliver_subject` - the subject to deliver observed messages. Not allowed for pull subscriptions.
A delivery subject is required for queue subscribing as it configures a subject that all the queue
consumers should listen on.
* `:description` - a short description of the purpose of this customer.
* `:durable_name` - the name of the consumer, which the server will track, allowing resuming consumption
where left off. By default, a consumer is ephemeral. To make the consumer durable, set the name.
See [naming](https://docs.nats.io/running-a-nats-service/nats_admin/jetstream_admin/naming).
* `:filter_subject` - when consuming from a stream with a wildcard subject, this allows you to select
a subset of the full wildcard subject to receive messages from.
* `:flow_control` - when set to true, an empty message with Status header 100 and a reply subject will
be sent. Consumers must reply to these messages to control the rate of message delivery.
* `:headers_only` - delivers only the headers of messages in the stream and not the bodies. Additionally
adds the Nats-Msg-Size header to indicate the size of the removed payload.
* `:idle_heartbeat` - if set, the server will regularly send a status message to the client while there
are no new messages to send. This lets the client know that the JetStream service is still up and
running, even when there is no activity on the stream. The message status header will have a code of 100.
Unlike `:flow_control`, it will have no reply to address. It may have a description like
"Idle Heartbeat".
* `:inactive_threshold` - duration that instructs the server to clean up ephemeral consumers that are
inactive for that long.
* `:max_ack_pending` - it sets the maximum number of messages without an acknowledgement that can be
outstanding, once this limit is reached, message delivery will be suspended. It cannot be used with
`:ack_none` ack policy. This maximum number of pending acks applies for all the consumer's
subscriber processes. A value of -1 means there can be any number of pending acks (i.e. no flow
control).
* `:max_batch` - the largest batch property that may be specified when doing a pull on a Pull consumer.
* `:max_deliver` - the maximum number of times a specific message will be delivered. Applies to any
message that is re-sent due to ack policy.
* `:max_expires` - the maximum expires value that may be set when doing a pull on a Pull consumer.
* `:max_waiting` - the number of pulls that can be outstanding on a pull consumer, pulls received after
this is reached are ignored.
* `:opt_start_seq` - use with `:deliver_policy` set to `:by_start_sequence`. It represents the sequence
number to start consuming on.
* `:opt_start_time` - use with `:deliver_policy` set to `:by_start_time`. It represents the time to start
consuming at.
* `:rate_limit_bps` - used to throttle the delivery of messages to the consumer, in bits per second.
* `:replay_policy` - it applies when the `:deliver_policy` is set to `:all`, `:by_start_sequence` or
`:by_start_time`. It has the following options:
- `:instant` - the default policy. The messages will be pushed to the client as fast as possible.
- `:original` - the messages in the stream will be pushed to the client at the same rate that they
were originally received.
* `:sample_freq` - Sets the percentage of acknowledgements that should be sampled for observability, 0-100.
This value is a binary and for example allows both `30` and `30%` as valid values.
"""
import Gnat.Jetstream.API.Util
@enforce_keys [:stream_name]
defstruct [
:backoff,
:deliver_group,
:deliver_subject,
:description,
:domain,
:durable_name,
:filter_subject,
:flow_control,
:headers_only,
:idle_heartbeat,
:inactive_threshold,
:max_batch,
:max_expires,
:max_waiting,
:opt_start_seq,
:opt_start_time,
:rate_limit_bps,
:sample_freq,
:stream_name,
ack_policy: :explicit,
ack_wait: 30_000_000_000,
deliver_policy: :all,
max_ack_pending: 20_000,
max_deliver: -1,
replay_policy: :instant
]
@type t :: %__MODULE__{
stream_name: binary(),
domain: nil | binary(),
ack_policy: :none | :all | :explicit,
ack_wait: nil | non_neg_integer(),
backoff: nil | [non_neg_integer()],
deliver_group: nil | binary(),
deliver_policy:
:all | :last | :new | :by_start_sequence | :by_start_time | :last_per_subject,
deliver_subject: nil | binary(),
description: nil | binary(),
durable_name: nil | binary(),
filter_subject: nil | binary(),
flow_control: nil | boolean(),
headers_only: nil | boolean(),
idle_heartbeat: nil | non_neg_integer(),
inactive_threshold: nil | non_neg_integer(),
max_ack_pending: nil | integer(),
max_batch: nil | integer(),
max_deliver: nil | integer(),
max_expires: nil | non_neg_integer(),
max_waiting: nil | integer(),
opt_start_seq: nil | non_neg_integer(),
opt_start_time: nil | DateTime.t(),
rate_limit_bps: nil | non_neg_integer(),
replay_policy: :instant | :original,
sample_freq: nil | binary()
}
@type info :: %{
ack_floor: %{
consumer_seq: non_neg_integer(),
stream_seq: non_neg_integer()
},
cluster:
nil
| %{
optional(:name) => binary(),
optional(:leader) => binary(),
optional(:replicas) => [
%{
:active => non_neg_integer(),
:current => boolean(),
:name => binary(),
optional(:lag) => non_neg_integer(),
optional(:offline) => boolean()
}
]
},
config: config(),
created: DateTime.t(),
delivered: %{
consumer_seq: non_neg_integer(),
stream_seq: non_neg_integer()
},
name: binary(),
num_ack_pending: non_neg_integer(),
num_pending: non_neg_integer(),
num_redelivered: non_neg_integer(),
num_waiting: non_neg_integer(),
push_bound: nil | boolean(),
stream_name: binary()
}
@type config :: %{
ack_policy: :none | :all | :explicit,
ack_wait: nil | non_neg_integer(),
backoff: nil | [non_neg_integer()],
deliver_group: nil | binary(),
deliver_policy:
:all | :last | :new | :by_start_sequence | :by_start_time | :last_per_subject,
deliver_subject: nil | binary(),
description: nil | binary(),
durable_name: nil | binary(),
filter_subject: nil | binary(),
flow_control: nil | boolean(),
headers_only: nil | boolean(),
idle_heartbeat: nil | non_neg_integer(),
inactive_threshold: nil | non_neg_integer(),
max_ack_pending: nil | integer(),
max_batch: nil | integer(),
max_deliver: nil | integer(),
max_expires: nil | non_neg_integer(),
max_waiting: nil | integer(),
opt_start_seq: nil | non_neg_integer(),
opt_start_time: nil | DateTime.t(),
rate_limit_bps: nil | non_neg_integer(),
replay_policy: :instant | :original,
sample_freq: nil | binary()
}
@type consumers :: %{
consumers: list(binary()),
limit: non_neg_integer(),
offset: non_neg_integer(),
total: non_neg_integer()
}
@doc """
Creates a consumer. When consumer's `:durable_name` field is not set, the function
creates an ephemeral consumer. Otherwise, it creates a durable consumer.
## Examples
iex> {:ok, _response} = Gnat.Jetstream.API.Stream.create(:gnat, %Gnat.Jetstream.API.Stream{name: "astream", subjects: ["subject"]})
iex> {:ok, %{name: "consumer", stream_name: "astream"}} = Gnat.Jetstream.API.Consumer.create(:gnat, %Gnat.Jetstream.API.Consumer{durable_name: "consumer", stream_name: "astream"})
iex> {:ok, _response} = Gnat.Jetstream.API.Stream.create(:gnat, %Gnat.Jetstream.API.Stream{name: "astream", subjects: ["subject"]})
iex> {:error, %{"description" => "consumer delivery policy is deliver by start sequence, but optional start sequence is not set"}} = Gnat.Jetstream.API.Consumer.create(:gnat, %Gnat.Jetstream.API.Consumer{durable_name: "consumer", stream_name: "astream", deliver_policy: :by_start_sequence})
"""
@spec create(conn :: Gnat.t(), consumer :: t()) :: {:ok, info()} | {:error, term()}
def create(conn, %__MODULE__{durable_name: name} = consumer) when not is_nil(name) do
create_topic =
"#{js_api(consumer.domain)}.CONSUMER.DURABLE.CREATE.#{consumer.stream_name}.#{name}"
with :ok <- validate_durable(consumer),
{:ok, raw_response} <- request(conn, create_topic, create_payload(consumer)) do
{:ok, to_info(raw_response)}
end
end
def create(conn, %__MODULE__{} = consumer) do
create_topic = "#{js_api(consumer.domain)}.CONSUMER.CREATE.#{consumer.stream_name}"
with :ok <- validate(consumer),
{:ok, raw_response} <- request(conn, create_topic, create_payload(consumer)) do
{:ok, to_info(raw_response)}
end
end
@doc """
Deletes a consumer.
## Examples
iex> {:ok, _response} = Gnat.Jetstream.API.Stream.create(:gnat, %Gnat.Jetstream.API.Stream{name: "astream", subjects: ["subject"]})
iex> {:ok, _response} = Gnat.Jetstream.API.Consumer.create(:gnat, %Gnat.Jetstream.API.Consumer{durable_name: "consumer", stream_name: "astream"})
iex> Gnat.Jetstream.API.Consumer.delete(:gnat, "astream", "consumer")
:ok
iex> {:error, %{"code" => 404, "description" => "stream not found"}} = Gnat.Jetstream.API.Consumer.delete(:gnat, "wrong_stream", "consumer")
"""
@spec delete(
conn :: Gnat.t(),
stream_name :: binary(),
consumer_name :: binary(),
domain :: nil | binary()
) ::
:ok | {:error, any()}
def delete(conn, stream_name, consumer_name, domain \\ nil) do
topic = "#{js_api(domain)}.CONSUMER.DELETE.#{stream_name}.#{consumer_name}"
with {:ok, _response} <- request(conn, topic, "") do
:ok
end
end
@doc """
Information about the consumer.
## Examples
iex> {:ok, _response} = Gnat.Jetstream.API.Stream.create(:gnat, %Gnat.Jetstream.API.Stream{name: "astream", subjects: ["subject"]})
iex> {:ok, _response} = Gnat.Jetstream.API.Consumer.create(:gnat, %Gnat.Jetstream.API.Consumer{durable_name: "consumer", stream_name: "astream"})
iex> {:ok, %{created: _}} = Gnat.Jetstream.API.Consumer.info(:gnat, "astream", "consumer")
iex> {:error, %{"code" => 404, "description" => "stream not found"}} = Gnat.Jetstream.API.Consumer.info(:gnat, "wrong_stream", "consumer")
"""
@spec info(
conn :: Gnat.t(),
stream_name :: binary(),
consumer_name :: binary(),
domain :: nil | binary()
) ::
{:ok, info()} | {:error, any()}
def info(conn, stream_name, consumer_name, domain \\ nil) do
topic = "#{js_api(domain)}.CONSUMER.INFO.#{stream_name}.#{consumer_name}"
with {:ok, raw} <- request(conn, topic, "") do
{:ok, to_info(raw)}
end
end
@doc """
Paged list of known consumers, including their current info.
## Examples
iex> {:ok, _response} = Gnat.Jetstream.API.Stream.create(:gnat, %Gnat.Jetstream.API.Stream{name: "astream", subjects: ["subject"]})
iex> {:ok, %{consumers: _, limit: 1024, offset: 0, total: _}} = Gnat.Jetstream.API.Consumer.list(:gnat, "astream")
iex> {:error, %{"code" => 404, "description" => "stream not found"}} = Gnat.Jetstream.API.Consumer.list(:gnat, "wrong_stream")
"""
@spec list(
conn :: Gnat.t(),
stream_name :: binary(),
params :: [offset: non_neg_integer(), domain: nil | binary()]
) ::
{:ok, consumers()} | {:error, term()}
def list(conn, stream_name, params \\ []) do
domain = Keyword.get(params, :domain)
payload =
Jason.encode!(%{
offset: Keyword.get(params, :offset, 0)
})
with {:ok, raw} <- request(conn, "#{js_api(domain)}.CONSUMER.NAMES.#{stream_name}", payload) do
response = %{
consumers: Map.get(raw, "consumers"),
limit: Map.get(raw, "limit"),
offset: Map.get(raw, "offset"),
total: Map.get(raw, "total")
}
{:ok, response}
end
end
@doc """
Requests a next message from a stream to be consumed. The response (consumed message)will be sent
on the subject given as the `reply_to` parameter.
## Options
* `batch` - How many messages to receive. Messages will be sent to the `reply_to` subject
separately. Defaults to 1.
* `expires` - Time in nanoseconds the request will be kept in the server. Once this time passes
a message with empty body and topic set to `reply_to` subject is sent. Useful when polling
the server frequently and not wanting the pull requests to accumulate. By default, the pull
request stays in the server until a message comes.
* `no_wait` - Boolean value which indicates whether the pull request should be accumulated on
the server. When set to true and no message is present to be consumed, a message with empty
body and topic value set to `reply_to` is sent. Defaults to false.
## Example
iex> {:ok, _response} = Gnat.Jetstream.API.Stream.create(:gnat, %Gnat.Jetstream.API.Stream{name: "astream", subjects: ["subject"]})
iex> {:ok, _response} = Gnat.Jetstream.API.Consumer.create(:gnat, %Gnat.Jetstream.API.Consumer{durable_name: "consumer", stream_name: "astream"})
iex> {:ok, _sid} = Gnat.sub(:gnat, self(), "reply_subject")
iex> :ok = Gnat.Jetstream.API.Consumer.request_next_message(:gnat, "astream", "consumer", "reply_subject")
iex> :ok = Gnat.pub(:gnat, "subject", "message1")
iex> assert_receive {:msg, %{body: "message1", topic: "subject"}}
"""
@spec request_next_message(
conn :: Gnat.t(),
stream_name :: binary(),
consumer_name :: binary(),
reply_to :: String.t(),
domain :: nil | binary(),
opts :: keyword()
) :: :ok
def request_next_message(
conn,
stream_name,
consumer_name,
reply_to,
domain \\ nil,
opts \\ []
) do
default_payload = %{batch: 1}
put_option_if_not_nil = fn payload, option_key ->
if option_value = opts[option_key] do
Map.put(payload, option_key, option_value)
else
payload
end
end
payload =
default_payload
|> put_option_if_not_nil.(:batch)
|> put_option_if_not_nil.(:no_wait)
|> put_option_if_not_nil.(:expires)
|> Jason.encode!()
Gnat.pub(
conn,
"#{js_api(domain)}.CONSUMER.MSG.NEXT.#{stream_name}.#{consumer_name}",
payload,
reply_to: reply_to
)
end
# https://docs.nats.io/running-a-nats-service/configuration/leafnodes/jetstream_leafnodes
defp js_api(nil), do: "$JS.API"
defp js_api(""), do: "$JS.API"
defp js_api(domain), do: "$JS.#{domain}.API"
defp create_payload(%__MODULE__{} = cons) do
%{
config: %{
ack_policy: cons.ack_policy,
ack_wait: cons.ack_wait,
backoff: cons.backoff,
deliver_group: cons.deliver_group,
deliver_policy: cons.deliver_policy,
deliver_subject: cons.deliver_subject,
description: cons.description,
durable_name: cons.durable_name,
filter_subject: cons.filter_subject,
flow_control: cons.flow_control,
headers_only: cons.headers_only,
idle_heartbeat: cons.idle_heartbeat,
inactive_threshold: cons.inactive_threshold,
max_ack_pending: cons.max_ack_pending,
max_batch: cons.max_batch,
max_deliver: cons.max_deliver,
max_expires: cons.max_expires,
max_waiting: cons.max_waiting,
opt_start_seq: cons.opt_start_seq,
opt_start_time: cons.opt_start_time,
rate_limit_bps: cons.rate_limit_bps,
replay_policy: cons.replay_policy,
sample_freq: cons.sample_freq
},
stream_name: cons.stream_name
}
|> Jason.encode!()
end
defp to_config(raw) do
%{
ack_policy: raw |> Map.get("ack_policy") |> to_sym(),
ack_wait: raw |> Map.get("ack_wait"),
backoff: Map.get(raw, "backoff"),
deliver_group: Map.get(raw, "deliver_group"),
deliver_policy: raw |> Map.get("deliver_policy") |> to_sym(),
deliver_subject: raw |> Map.get("deliver_subject"),
description: Map.get(raw, "description"),
durable_name: Map.get(raw, "durable_name"),
filter_subject: raw |> Map.get("filter_subject"),
flow_control: Map.get(raw, "flow_control"),
headers_only: Map.get(raw, "headers_only"),
idle_heartbeat: Map.get(raw, "idle_heartbeat"),
inactive_threshold: Map.get(raw, "inactive_threshold"),
max_ack_pending: Map.get(raw, "max_ack_pending"),
max_batch: Map.get(raw, "max_batch"),
max_deliver: Map.get(raw, "max_deliver"),
max_expires: Map.get(raw, "max_expires"),
max_waiting: Map.get(raw, "max_waiting"),
opt_start_seq: raw |> Map.get("opt_start_seq"),
opt_start_time: raw |> Map.get("opt_start_time") |> to_datetime(),
rate_limit_bps: Map.get(raw, "rate_limit_bps"),
replay_policy: raw |> Map.get("replay_policy") |> to_sym(),
sample_freq: Map.get(raw, "sample_freq")
}
end
defp to_info(raw) do
%{
ack_floor: %{
consumer_seq: get_in(raw, ["ack_floor", "consumer_seq"]),
stream_seq: get_in(raw, ["ack_floor", "stream_seq"])
},
cluster: Map.get(raw, "cluster"),
config: to_config(Map.get(raw, "config")),
created: raw |> Map.get("created") |> to_datetime(),
delivered: %{
consumer_seq: get_in(raw, ["delivered", "consumer_seq"]),
stream_seq: get_in(raw, ["delivered", "stream_seq"])
},
name: Map.get(raw, "name"),
num_ack_pending: Map.get(raw, "num_ack_pending"),
num_pending: Map.get(raw, "num_pending"),
num_redelivered: Map.get(raw, "num_redelivered"),
num_waiting: Map.get(raw, "num_waiting"),
push_bound: Map.get(raw, "push_bound"),
stream_name: Map.get(raw, "stream_name")
}
end
defp validate(consumer) do
cond do
consumer.stream_name == nil ->
{:error, "must have a :stream_name set"}
is_binary(consumer.stream_name) == false ->
{:error, "stream_name must be a string"}
valid_name?(consumer.stream_name) == false ->
{:error, "invalid stream_name: " <> invalid_name_message()}
consumer.deliver_policy not in [
:all,
:last,
:new,
:by_start_sequence,
:by_start_time,
:last_per_subject
] ->
{:error, "invalid deliver policy: #{consumer.deliver_policy}"}
consumer.replay_policy not in [:instant, :original] ->
{:error, "invalid replay policy: #{consumer.replay_policy}"}
true ->
:ok
end
end
defp validate_durable(consumer) do
with :ok <- validate(consumer) do
cond do
is_binary(consumer.durable_name) == false ->
{:error, "durable_name must be a string"}
valid_name?(consumer.durable_name) == false ->
{:error, "invalid durable_name: " <> invalid_name_message()}
true ->
:ok
end
end
end
end
================================================
FILE: lib/gnat/jetstream/api/kv/entry.ex
================================================
defmodule Gnat.Jetstream.API.KV.Entry do
@moduledoc """
A parsed view of a single message from a Key/Value bucket's underlying stream.
Messages delivered from a KV bucket's stream encode three different operations
(put, delete, purge) using a combination of the `kv-operation` header, the
`nats-marker-reason` header, and the absence of any headers. Recovering the
original key also requires stripping the `$KV.<bucket>.` subject prefix.
This module captures that convention in one place so both the built-in
`Gnat.Jetstream.API.KV.Watcher` (push consumer) and user-supplied
`Gnat.Jetstream.PullConsumer` implementations can share it.
## Using with a custom PullConsumer
A common use case is hydrating a local cache from a KV bucket by driving a
`Gnat.Jetstream.PullConsumer`. Inside `c:handle_message/2`, convert the raw
message into an `Entry` and branch on the operation:
defmodule MyApp.KVCache do
use Gnat.Jetstream.PullConsumer
alias Gnat.Jetstream.API.KV
@bucket "my_bucket"
@impl true
def handle_message(message, state) do
case KV.Entry.from_message(message, @bucket) do
{:ok, %KV.Entry{operation: :put, key: key, value: value}} ->
{:ack, put_in(state.cache[key], value)}
{:ok, %KV.Entry{operation: op, key: key}} when op in [:delete, :purge] ->
{:ack, update_in(state.cache, &Map.delete(&1, key))}
:ignore ->
{:ack, state}
end
end
end
The returned struct also carries the JetStream `revision` (stream sequence),
`created` timestamp, and `delta` (`num_pending`) when the message includes
JetStream metadata, which is useful for detecting when the consumer has
caught up with the tail of the stream (`delta == 0`).
## Messages that are not KV records
`from_message/2` returns `:ignore` when the input is not a KV record — for
example a JetStream status message (`100` heartbeat, `404`/`408` pull
terminator, `409` leadership change) or a message whose subject does not
belong to the given bucket. In normal operation the `Watcher` and
`PullConsumer` layers filter status messages out before they reach user
code, so this is a defensive fallback rather than something consumers are
expected to rely on.
"""
alias Gnat.Jetstream.API.Message
@operation_header "kv-operation"
@operation_del "DEL"
@operation_purge "PURGE"
@nats_marker_reason_header "nats-marker-reason"
@subject_prefix "$KV."
@type operation :: :put | :delete | :purge
@type t :: %__MODULE__{
bucket: String.t(),
key: String.t(),
value: binary(),
operation: operation(),
revision: non_neg_integer() | nil,
created: DateTime.t() | nil,
delta: non_neg_integer() | nil
}
defstruct [:bucket, :key, :value, :operation, :revision, :created, :delta]
@doc """
Parse a NATS message delivered from a KV bucket's underlying stream into an
`Entry`.
`bucket_name` must match the bucket the message was published to; it is used
to strip the `$KV.<bucket>.` subject prefix and recover the key.
Returns `:ignore` if the message is not a KV record for the given bucket
(JetStream status message, wrong subject, etc.).
The `:revision`, `:created`, and `:delta` fields are populated when the
message carries a JetStream `$JS.ACK...` reply subject. For messages without
one (e.g. direct get responses), those fields are `nil`.
"""
@spec from_message(Gnat.message(), bucket_name :: String.t()) :: {:ok, t()} | :ignore
def from_message(message, bucket_name) do
with false <- status_message?(message),
{:ok, key} <- extract_key(message, bucket_name) do
entry = %__MODULE__{
bucket: bucket_name,
key: key,
value: Map.get(message, :body, ""),
operation: operation(message)
}
{:ok, apply_metadata(entry, message)}
else
_ -> :ignore
end
end
defp status_message?(%{status: status}) when is_binary(status) and status != "", do: true
defp status_message?(_), do: false
defp extract_key(%{topic: topic}, bucket_name) do
prefix = @subject_prefix <> bucket_name <> "."
if String.starts_with?(topic, prefix) do
{:ok, binary_part(topic, byte_size(prefix), byte_size(topic) - byte_size(prefix))}
else
:error
end
end
defp operation(%{headers: headers}) when is_list(headers) do
Enum.find_value(headers, :put, fn
{@operation_header, @operation_del} -> :delete
{@operation_header, @operation_purge} -> :purge
{@nats_marker_reason_header, _} -> :delete
_ -> false
end)
end
defp operation(_message), do: :put
defp apply_metadata(entry, message) do
case Message.metadata(message) do
{:ok, metadata} ->
%__MODULE__{
entry
| revision: metadata.stream_seq,
created: metadata.timestamp,
delta: metadata.num_pending
}
{:error, _} ->
entry
end
end
end
================================================
FILE: lib/gnat/jetstream/api/kv/watcher.ex
================================================
defmodule Gnat.Jetstream.API.KV.Watcher do
@moduledoc """
The watcher server establishes a subscription to the changes that occur to a given key-value bucket. The
consumer-supplied handler function will be sent an indicator as to whether the change is a delete or an add,
as well as the key being changed and the value (if it was added).
Ensure that you call `stop` with a watcher pid when you no longer need to be notified about key changes
"""
use GenServer
alias Gnat.Jetstream.API.{Consumer, KV, Util}
alias Gnat.Jetstream.API.KV.Entry
# Matches the ordered-consumer defaults used by the nats.go KV watcher:
# a 5s idle heartbeat and server-driven flow control. The flow-control
# messages arrive as 100-status messages with a reply subject — the client
# is expected to publish an empty reply so the server releases backpressure.
@flow_control_heartbeat_ns 5_000_000_000
@type keywatch_handler ::
(action :: :key_deleted | :key_added, key :: String.t(), value :: any() -> nil)
@type watcher_options ::
{:conn, Gnat.t()}
| {:bucket_name, String.t()}
| {:handler, keywatch_handler()}
@spec start_link(opts :: [watcher_options()]) :: GenServer.on_start()
def start_link(opts) do
GenServer.start_link(__MODULE__, opts)
end
def stop(pid) do
GenServer.stop(pid)
end
def init(opts) do
{:ok, {sub, consumer_name}} = subscribe(opts[:conn], opts[:bucket_name])
{:ok,
%{
handler: opts[:handler],
conn: opts[:conn],
bucket_name: opts[:bucket_name],
sub: sub,
consumer_name: consumer_name,
domain: Keyword.get(opts, :domain)
}}
end
def terminate(_reason, state) do
stream = KV.stream_name(state.bucket_name)
:ok = Gnat.unsub(state.conn, state.sub)
:ok = Consumer.delete(state.conn, stream, state.consumer_name, state.domain)
end
# Flow-control request: the server is asking us to acknowledge that we're
# keeping up. Responding releases backpressure so the server continues
# delivering messages to slow handlers rather than dropping us as a slow
# consumer.
def handle_info({:msg, %{status: "100", reply_to: reply_to}}, state)
when is_binary(reply_to) and reply_to != "" do
:ok = Gnat.pub(state.conn, reply_to, "")
{:noreply, state}
end
# Idle heartbeat (status 100 with no reply_to) and any other informational
# status message (404, 408, 409, etc.) — not a stream record, drop it.
def handle_info({:msg, %{status: status}}, state)
when is_binary(status) and status != "" do
{:noreply, state}
end
def handle_info({:msg, message}, state) do
case Entry.from_message(message, state.bucket_name) do
{:ok, entry} ->
state.handler.(action(entry.operation), entry.key, entry.value)
:ignore ->
:ok
end
{:noreply, state}
end
defp action(:put), do: :key_added
defp action(:delete), do: :key_deleted
defp action(:purge), do: :key_purged
defp subscribe(conn, bucket_name) do
stream = KV.stream_name(bucket_name)
inbox = Util.reply_inbox()
consumer_name = "all_key_values_watcher_#{Util.nuid()}"
with {:ok, sub} <- Gnat.sub(conn, self(), inbox),
{:ok, _consumer} <-
Consumer.create(conn, %Consumer{
durable_name: consumer_name,
deliver_subject: inbox,
stream_name: stream,
ack_policy: :none,
max_ack_pending: -1,
max_deliver: 1,
flow_control: true,
idle_heartbeat: @flow_control_heartbeat_ns
}) do
{:ok, {sub, consumer_name}}
end
end
end
================================================
FILE: lib/gnat/jetstream/api/kv.ex
================================================
defmodule Gnat.Jetstream.API.KV do
@moduledoc """
API for interacting with the Key/Value store functionality in Nats Jetstream.
Learn about the Key/Value store: https://docs.nats.io/nats-concepts/jetstream/key-value-store
## Consuming KV changes from a custom PullConsumer
`watch/3` covers the common "push consumer" use case for reacting to bucket
changes. If you need a pull-based consumer instead (e.g. to hydrate a local
cache and detect when it is caught up with the stream), see
`Gnat.Jetstream.API.KV.Entry` for a shared helper that parses a raw NATS
message into a KV operation/key/value triple with optional revision
metadata.
"""
alias Gnat.Jetstream.API.{Stream}
@stream_prefix "KV_"
@subject_prefix "$KV."
@two_minutes_in_nanoseconds 120_000_000_000
@type bucket_options ::
{:history, non_neg_integer()}
| {:ttl, non_neg_integer()}
| {:limit_marker_ttl, non_neg_integer()}
| {:max_bucket_size, non_neg_integer()}
| {:max_value_size, non_neg_integer()}
| {:description, binary()}
| {:replicas, non_neg_integer()}
| {:storage, :file | :memory}
| {:placement, Stream.placement()}
@doc """
Create a new Key/Value bucket. Can include the following options
* `:history` - How many historic values to keep per key (defaults to 1, max of 64)
* `:ttl` - How long to keep values for (in nanoseconds)
* `:limit_marker_ttl` - How long the bucket keeps markers when keys are removed by the TTL setting.
* `:max_bucket_size` - The max number of bytes the bucket can hold
* `:max_value_size` - The max number of bytes a value may be
* `:description` - A description for the bucket
* `:replicas` - How many replicas of the data to store
* `:storage` - Storage backend to use (:file, :memory)
* `:placement` - A map with :cluster (required) and :tags (optional)
## Examples
iex> {:ok, info} = Jetstream.API.KV.create_bucket(:gnat, "my_bucket")
"""
@spec create_bucket(conn :: Gnat.t(), bucket_name :: binary(), params :: [bucket_options()]) ::
{:ok, Stream.info()} | {:error, any()}
def create_bucket(conn, bucket_name, params \\ []) do
# The primary NATS docs don't provide information about how to interact
# with Key-Value functionality over the wire. Turns out the KV store is
# just a Stream under-the-hood
# Discovered these settings from looking at the `nats-server -js -DV` logs
# as well as the GoLang implementation https://github.com/nats-io/nats.go/blob/dd91b86bc4f7fa0f061fefe11506aaee413bfafd/kv.go#L339
# If the settings aren't correct, NATS will not consider it a valid KV store
stream = %Stream{
name: stream_name(bucket_name),
subjects: stream_subjects(bucket_name),
description: Keyword.get(params, :description),
max_msgs_per_subject: Keyword.get(params, :history, 1),
discard: :new,
deny_delete: true,
allow_rollup_hdrs: true,
max_age: Keyword.get(params, :ttl, 0),
max_bytes: Keyword.get(params, :max_bucket_size, -1),
max_msg_size: Keyword.get(params, :max_value_size, -1),
num_replicas: Keyword.get(params, :replicas, 1),
storage: Keyword.get(params, :storage, :file),
placement: Keyword.get(params, :placement),
duplicate_window: adjust_duplicate_window(Keyword.get(params, :ttl, 0)),
subject_delete_marker_ttl: Keyword.get(params, :limit_marker_ttl, 0)
}
Stream.create(conn, stream)
end
# The `duplicate_window` can't be greater than the `max_age`. The default `duplicate_window`
# is 2 minutes. We'll keep the 2 minute window UNLESS the ttl is less than 2 minutes
defp adjust_duplicate_window(ttl) when ttl > 0 and ttl < @two_minutes_in_nanoseconds, do: ttl
defp adjust_duplicate_window(_ttl), do: @two_minutes_in_nanoseconds
@doc """
Delete a Key/Value bucket
## Examples
iex> :ok = Jetstream.API.KV.delete_bucket(:gnat, "my_bucket")
"""
@spec delete_bucket(conn :: Gnat.t(), bucket_name :: binary()) :: :ok | {:error, any()}
def delete_bucket(conn, bucket_name) do
Stream.delete(conn, stream_name(bucket_name))
end
@doc """
Create a Key in a Key/Value Bucket
## Options
* `:timeout` - receive timeout for the request
## Examples
iex> :ok = Jetstream.API.KV.create_key(:gnat, "my_bucket", "my_key", "my_value")
"""
@spec create_key(
conn :: Gnat.t(),
bucket_name :: binary(),
key :: binary(),
value :: binary(),
opts :: keyword()
) ::
:ok | {:error, any()}
def create_key(conn, bucket_name, key, value, opts \\ []) do
timeout = Keyword.get(opts, :timeout, 5_000)
reply = Gnat.request(conn, key_name(bucket_name, key), value, receive_timeout: timeout)
case reply do
{:ok, _} -> :ok
error -> error
end
end
@doc """
Delete a Key from a K/V Bucket
## Examples
iex> :ok = Jetstream.API.KV.delete_key(:gnat, "my_bucket", "my_key")
"""
@spec delete_key(
conn :: Gnat.t(),
bucket_name :: binary(),
key :: binary(),
opts :: keyword()
) ::
:ok | {:error, any()}
def delete_key(conn, bucket_name, key, opts \\ []) do
timeout = Keyword.get(opts, :timeout, 5_000)
reply =
Gnat.request(conn, key_name(bucket_name, key), "",
headers: [{"KV-Operation", "DEL"}],
receive_timeout: timeout
)
case reply do
{:ok, _} -> :ok
error -> error
end
end
@doc """
Purge a Key from a K/V bucket. This will remove any revision history the key had
## Examples
iex> :ok = Jetstream.API.KV.purge_key(:gnat, "my_bucket", "my_key")
"""
@spec purge_key(
conn :: Gnat.t(),
bucket_name :: binary(),
key :: binary(),
opts :: keyword()
) ::
:ok | {:error, any()}
def purge_key(conn, bucket_name, key, opts \\ []) do
timeout = Keyword.get(opts, :timeout, 5_000)
reply =
Gnat.request(conn, key_name(bucket_name, key), "",
headers: [{"KV-Operation", "PURGE"}, {"Nats-Rollup", "sub"}],
receive_timeout: timeout
)
case reply do
{:ok, _} -> :ok
error -> error
end
end
@doc """
Put a value into a Key in a K/V Bucket
## Examples
iex> :ok = Jetstream.API.KV.put_value(:gnat, "my_bucket", "my_key", "my_value")
"""
@spec put_value(
conn :: Gnat.t(),
bucket_name :: binary(),
key :: binary(),
value :: binary(),
opts :: keyword()
) ::
:ok | {:error, any()}
def put_value(conn, bucket_name, key, value, opts \\ []) do
timeout = Keyword.get(opts, :timeout, 5_000)
reply = Gnat.request(conn, key_name(bucket_name, key), value, receive_timeout: timeout)
case reply do
{:ok, _} -> :ok
error -> error
end
end
@doc """
Get the value for a key in a particular K/V bucket
## Examples
iex> "my_value" = Jetstream.API.KV.get_value(:gnat, "my_bucket", "my_key")
"""
@spec get_value(conn :: Gnat.t(), bucket_name :: binary(), key :: binary()) ::
binary() | {:error, any()} | nil
def get_value(conn, bucket_name, key) do
case Stream.get_message(conn, stream_name(bucket_name), %{
last_by_subj: key_name(bucket_name, key)
}) do
{:ok, message} -> message.data
error -> error
end
end
@doc """
Get all the non-deleted key-value pairs for a Bucket
## Options
* `:batch` - Number of messages to fetch per request (default: 250)
* `:domain` - JetStream domain (default: nil)
## Examples
iex> {:ok, %{"key1" => "value1"}} = Jetstream.API.KV.contents(:gnat, "my_bucket")
iex> {:ok, contents} = Jetstream.API.KV.contents(:gnat, "my_bucket", batch: 500)
"""
@spec contents(conn :: Gnat.t(), bucket_name :: binary(), opts :: keyword()) ::
{:ok, map()} | {:error, binary()}
def contents(conn, bucket_name, opts \\ []) do
alias Gnat.Jetstream.Pager
stream = stream_name(bucket_name)
domain = Keyword.get(opts, :domain)
batch_size = Keyword.get(opts, :batch, 250)
pager_opts = [domain: domain, batch: batch_size]
Pager.reduce(conn, stream, pager_opts, %{}, fn msg, acc ->
case msg do
%{topic: key, body: body, headers: headers} ->
if {"kv-operation", "DEL"} in headers do
acc
else
Map.put(acc, subject_to_key(key, bucket_name), body)
end
%{topic: key, body: body} ->
Map.put(acc, subject_to_key(key, bucket_name), body)
end
end)
end
@doc """
Get all the non-deleted keys for a Bucket
## Options
* `:batch` - Number of messages to fetch per request (default: 250)
* `:domain` - JetStream domain (default: nil)
## Examples
iex> {:ok, ["key1", "key2"]} = Jetstream.API.KV.keys(:gnat, "my_bucket")
iex> {:ok, keys} = Jetstream.API.KV.keys(:gnat, "my_bucket", batch: 500)
"""
@spec keys(conn :: Gnat.t(), bucket_name :: binary(), opts :: keyword()) ::
{:ok, list(binary())} | {:error, binary()}
def keys(conn, bucket_name, opts \\ []) do
alias Gnat.Jetstream.Pager
stream = stream_name(bucket_name)
domain = Keyword.get(opts, :domain)
batch_size = Keyword.get(opts, :batch, 250)
pager_opts = [domain: domain, headers_only: true, batch: batch_size]
result =
Pager.reduce(conn, stream, pager_opts, MapSet.new(), fn msg, acc ->
case msg do
%{topic: key, headers: headers} ->
cond do
{"kv-operation", "DEL"} in headers ->
MapSet.delete(acc, subject_to_key(key, bucket_name))
{"kv-operation", "PURGE"} in headers ->
MapSet.delete(acc, subject_to_key(key, bucket_name))
true ->
MapSet.put(acc, subject_to_key(key, bucket_name))
end
%{topic: key} ->
MapSet.put(acc, subject_to_key(key, bucket_name))
end
end)
case result do
{:ok, key_set} -> {:ok, Enum.sort(MapSet.to_list(key_set))}
error -> error
end
end
@doc """
Information about the state of the bucket's Stream.
## Opts
* `:domain` - (default `nil`) the domain of the bucket
"""
@spec info(conn :: Gnat.t(), bucket_name :: binary(), keyword()) ::
{:ok, Stream.info()} | {:error, any()}
def info(conn, bucket_name, opts \\ []) do
Stream.info(conn, @stream_prefix <> bucket_name, Keyword.get(opts, :domain))
end
@doc ~S"""
Starts a monitor for key changes in a given bucket. Supply a handler that will receive
key change notifications.
## Examples
iex> {:ok, _pid} = Jetstream.API.KV.watch(:gnat, "my_bucket", fn action, key, value ->
...> IO.puts("#{action} taken on #{key}")
...> end)
"""
def watch(conn, bucket_name, handler) do
Gnat.Jetstream.API.KV.Watcher.start_link(
conn: conn,
bucket_name: bucket_name,
handler: handler
)
end
@doc ~S"""
Stops a previously running monitor. This will unsubscribe from the key changes and remove the
ephemeral consumer
## Examples
iex> :ok = Jetstream.API.KV.unwatch(pid)
"""
def unwatch(pid) do
Gnat.Jetstream.API.KV.Watcher.stop(pid)
end
@spec is_kv_bucket_stream?(stream_name :: binary()) :: boolean()
@deprecated "Use Gnat.Jetstream.API.KV.kv_bucket_stream?/1 instead"
def is_kv_bucket_stream?(stream_name) do
kv_bucket_stream?(stream_name)
end
@doc """
Returns true if the provided stream is a KV bucket, false otherwise
## Parameters
* `stream_name` - the stream name to test
"""
@spec kv_bucket_stream?(stream_name :: binary()) :: boolean()
def kv_bucket_stream?(stream_name) do
String.starts_with?(stream_name, "KV_")
end
@doc """
Returns a list of all the buckets in the KV
"""
@spec list_buckets(conn :: Gnat.t()) :: {:error, term()} | {:ok, list(String.t())}
def list_buckets(conn) do
with {:ok, %{streams: streams}} <- Stream.list(conn) do
stream_names =
for bucket <- streams, kv_bucket_stream?(bucket) do
String.trim_leading(bucket, @stream_prefix)
end
{:ok, stream_names}
else
{:error, reason} ->
{:error, reason}
end
end
@doc false
def stream_name(bucket_name) do
"#{@stream_prefix}#{bucket_name}"
end
defp stream_subjects(bucket_name) do
["#{@subject_prefix}#{bucket_name}.>"]
end
defp key_name(bucket_name, key) do
"#{@subject_prefix}#{bucket_name}.#{key}"
end
@doc false
def subject_to_key(subject, bucket_name) do
String.replace(subject, "#{@subject_prefix}#{bucket_name}.", "")
end
end
================================================
FILE: lib/gnat/jetstream/api/message.ex
================================================
defmodule Gnat.Jetstream.API.Message do
@moduledoc """
This module provides a way to parse the `reply_to` received by a PullConsumer
and get some useful information about the state of the consumer.
"""
# Based on:
# https://github.com/nats-io/nats.py/blob/d9f24b4beae541b7723873ba0a786ea7c0ecb3d5/nats/aio/msg.py#L182
defmodule Metadata do
@type t :: %__MODULE__{
:stream_seq => integer(),
:consumer_seq => integer(),
:num_pending => integer(),
:num_delivered => integer(),
:timestamp => DateTime.t(),
:stream => String.t(),
:consumer => String.t(),
:domain => String.t() | nil
}
defstruct [
:stream_seq,
:consumer_seq,
:num_pending,
:num_delivered,
:timestamp,
:stream,
:consumer,
:domain
]
end
@spec metadata(message :: Gnat.message()) :: {:ok, Metadata.t()} | {:error, term()}
def metadata(%{reply_to: "$JS.ACK." <> ack_topic}),
do: decode_reply_to(String.split(ack_topic, "."))
def metadata(_), do: {:error, :no_jetstream_message}
# # Subject without domain:
# $JS.ACK.<stream>.<consumer>.<delivered>.<sseq>.<cseq>.<tm>.<pending>
defp decode_reply_to([
stream,
consumer,
num_delivered,
stream_seq,
consumer_seq,
ts,
num_pending
]) do
with {ts, ""} <- Integer.parse(ts),
{:ok, ts} <- DateTime.from_unix(ts, :nanosecond),
{stream_seq, ""} <- Integer.parse(stream_seq),
{consumer_seq, ""} <- Integer.parse(consumer_seq),
{num_delivered, ""} <- Integer.parse(num_delivered),
{num_pending, ""} <- Integer.parse(num_pending) do
{:ok,
%Metadata{
stream_seq: stream_seq,
consumer_seq: consumer_seq,
num_delivered: num_delivered,
num_pending: num_pending,
timestamp: ts,
stream: stream,
consumer: consumer
}}
else
_ ->
{:error, :invalid_ack_reply_to}
end
end
# Subject with domain:
# $JS.ACK.<domain>.<account hash>.<stream>.<consumer>.<delivered>.<sseq>.
# <cseq>.<tm>.<pending>.<a token with a random value>
defp decode_reply_to([
domain,
_account_hash,
stream,
consumer,
num_delivered,
stream_seq,
consumer_seq,
ts,
num_pending,
_random_value
]) do
case decode_reply_to([
stream,
consumer,
num_delivered,
stream_seq,
consumer_seq,
ts,
num_pending
]) do
{:ok, metadata} ->
domain = if domain == "_", do: nil, else: domain
{:ok, %{metadata | domain: domain}}
err = {:error, _} ->
err
end
end
defp decode_reply_to(_), do: {:error, :invalid_ack_reply_to}
end
================================================
FILE: lib/gnat/jetstream/api/object/meta.ex
================================================
defmodule Gnat.Jetstream.API.Object.Meta do
@enforce_keys [:bucket, :chunks, :digest, :name, :nuid, :size]
defstruct bucket: nil,
chunks: nil,
deleted: false,
digest: nil,
name: nil,
nuid: nil,
size: nil
@type t :: %__MODULE__{
bucket: String.t(),
chunks: non_neg_integer(),
deleted: boolean(),
digest: String.t(),
name: String.t(),
nuid: String.t(),
size: non_neg_integer()
}
end
defimpl Jason.Encoder, for: Gnat.Jetstream.API.Object.Meta do
alias Gnat.Jetstream.API.Object.Meta
def encode(%Meta{deleted: true} = meta, opts) do
Map.take(meta, [:bucket, :chunks, :deleted, :digest, :name, :nuid, :size])
|> Jason.Encode.map(opts)
end
def encode(meta, opts) do
Map.take(meta, [:bucket, :chunks, :digest, :name, :nuid, :size])
|> Jason.Encode.map(opts)
end
end
================================================
FILE: lib/gnat/jetstream/api/object.ex
================================================
defmodule Gnat.Jetstream.API.Object do
@moduledoc """
API for interacting with the JetStream Object Store
Learn more about Object Store: https://docs.nats.io/nats-concepts/jetstream/obj_store
"""
alias Gnat.Jetstream.API.{Consumer, Stream, Util}
alias Gnat.Jetstream.API.Object.Meta
@stream_prefix "OBJ_"
@subject_prefix "$O."
@type bucket_opt ::
{:description, String.t()}
| {:max_bucket_size, integer()}
| {:max_chunk_size, integer()}
| {:placement, Stream.placement()}
| {:replicas, non_neg_integer()}
| {:storage, :file | :memory}
| {:ttl, non_neg_integer()}
@spec create_bucket(Gnat.t(), String.t(), list(bucket_opt)) ::
{:ok, Stream.info()} | {:error, any()}
def create_bucket(conn, bucket_name, params \\ []) do
with :ok <- validate_bucket_name(bucket_name) do
stream = %Stream{
name: stream_name(bucket_name),
subjects: stream_subjects(bucket_name),
description: Keyword.get(params, :description),
discard: :new,
allow_rollup_hdrs: true,
max_age: Keyword.get(params, :ttl, 0),
max_bytes: Keyword.get(params, :max_bucket_size, -1),
max_msg_size: Keyword.get(params, :max_chunk_size, -1),
num_replicas: Keyword.get(params, :replicas, 1),
storage: Keyword.get(params, :storage, :file),
placement: Keyword.get(params, :placement),
duplicate_window: adjust_duplicate_window(Keyword.get(params, :ttl, 0))
}
Stream.create(conn, stream)
end
end
@spec delete_bucket(Gnat.t(), String.t()) :: :ok | {:error, any}
def delete_bucket(conn, bucket_name) do
Stream.delete(conn, stream_name(bucket_name))
end
@spec delete(Gnat.t(), String.t(), String.t()) :: :ok | {:error, any}
def delete(conn, bucket_name, object_name) do
with {:ok, meta = %Meta{}} <- info(conn, bucket_name, object_name),
meta <- %Meta{meta | deleted: true},
topic <- meta_stream_topic(bucket_name, object_name),
{:ok, body} <- Jason.encode(meta),
{:ok, _msg} <- Gnat.request(conn, topic, body, headers: [{"Nats-Rollup", "sub"}]) do
filter = chunk_stream_topic(meta)
Stream.purge(conn, stream_name(bucket_name), nil, %{filter: filter})
end
end
@spec get(Gnat.t(), String.t(), String.t(), (binary -> any())) :: :ok | {:error, any}
def get(conn, bucket_name, object_name, chunk_fun) do
with {:ok, %{config: _stream}} <- Stream.info(conn, stream_name(bucket_name)),
{:ok, meta} <- info(conn, bucket_name, object_name) do
receive_chunks(conn, meta, chunk_fun)
end
end
@spec info(Gnat.t(), String.t(), String.t()) :: {:ok, Meta.t()} | {:error, any}
def info(conn, bucket_name, object_name) do
with {:ok, _stream_info} <- Stream.info(conn, stream_name(bucket_name)) do
Stream.get_message(conn, stream_name(bucket_name), %{
last_by_subj: meta_stream_topic(bucket_name, object_name)
})
|> case do
{:ok, message} ->
meta = json_to_meta(message.data)
{:ok, meta}
error ->
error
end
end
end
@type list_option :: {:show_deleted, boolean()}
@spec list(Gnat.t(), String.t(), list(list_option())) :: {:error, any} | {:ok, list(Meta.t())}
def list(conn, bucket_name, options \\ []) do
with {:ok, %{config: stream}} <- Stream.info(conn, stream_name(bucket_name)),
topic <- Util.reply_inbox(),
{:ok, sub} <- Gnat.sub(conn, self(), topic),
{:ok, consumer} <-
Consumer.create(conn, %Consumer{
stream_name: stream.name,
deliver_subject: topic,
deliver_policy: :last_per_subject,
filter_subject: meta_stream_subject(bucket_name),
ack_policy: :none,
max_ack_pending: nil,
replay_policy: :instant,
max_deliver: 1
}),
{:ok, messages} <- receive_all_metas(sub, consumer.num_pending) do
:ok = Gnat.unsub(conn, sub)
:ok = Consumer.delete(conn, stream.name, consumer.name)
show_deleted = Keyword.get(options, :show_deleted, false)
if show_deleted do
{:ok, messages}
else
{:ok, Enum.reject(messages, &(&1.deleted == true))}
end
end
end
@spec put(Gnat.t(), String.t(), String.t(), File.io_device()) ::
{:ok, Meta.t()} | {:error, any()}
def put(conn, bucket_name, object_name, io) do
nuid = Util.nuid()
chunk_topic = chunk_stream_topic(bucket_name, nuid)
with {:ok, %{config: _}} <- Stream.info(conn, stream_name(bucket_name)),
:ok <- purge_prior_chunks(conn, bucket_name, object_name),
{:ok, chunks, size, digest} <- send_chunks(conn, io, chunk_topic) do
object_meta = %Meta{
name: object_name,
bucket: bucket_name,
nuid: nuid,
size: size,
chunks: chunks,
digest: "SHA-256=#{Base.url_encode64(digest)}"
}
topic = meta_stream_topic(bucket_name, object_name)
body = Jason.encode!(object_meta)
case Gnat.request(conn, topic, body, headers: [{"Nats-Rollup", "sub"}]) do
{:ok, _} ->
{:ok, object_meta}
error ->
error
end
end
end
@doc """
Returns true if the provided stream is an Object bucket, false otherwise
## Parameters
* `stream_name` - the stream name to test
"""
@spec is_object_bucket_stream?(stream_name :: binary()) :: boolean()
def is_object_bucket_stream?(stream_name) do
String.starts_with?(stream_name, "OBJ_")
end
@doc """
Returns a list of all Object buckets
"""
@spec list_buckets(conn :: Gnat.t()) :: {:error, term()} | {:ok, list(String.t())}
def list_buckets(conn) do
with {:ok, %{streams: streams}} <- Stream.list(conn) do
stream_names =
streams
|> Enum.flat_map(fn bucket ->
if is_object_bucket_stream?(bucket) do
[bucket |> String.trim_leading(@stream_prefix)]
else
[]
end
end)
{:ok, stream_names}
else
{:error, reason} ->
{:error, reason}
end
end
defp stream_name(bucket_name) do
"#{@stream_prefix}#{bucket_name}"
end
defp stream_subjects(bucket_name) do
[
chunk_stream_subject(bucket_name),
meta_stream_subject(bucket_name)
]
end
defp chunk_stream_subject(bucket_name) do
"#{@subject_prefix}#{bucket_name}.C.>"
end
defp chunk_stream_topic(bucket_name, nuid) do
"#{@subject_prefix}#{bucket_name}.C.#{nuid}"
end
defp chunk_stream_topic(%Meta{bucket: bucket, nuid: nuid}) do
"#{@subject_prefix}#{bucket}.C.#{nuid}"
end
defp meta_stream_subject(bucket_name) do
"#{@subject_prefix}#{bucket_name}.M.>"
end
defp meta_stream_topic(bucket_name, object_name) do
key = Base.url_encode64(object_name)
"#{@subject_prefix}#{bucket_name}.M.#{key}"
end
@two_minutes_in_nanoseconds 120_000_000_000
# The `duplicate_window` can't be greater than the `max_age`. The default `duplicate_window`
# is 2 minutes. We'll keep the 2 minute window UNLESS the ttl is less than 2 minutes
defp adjust_duplicate_window(ttl) when ttl > 0 and ttl < @two_minutes_in_nanoseconds, do: ttl
defp adjust_duplicate_window(_ttl), do: @two_minutes_in_nanoseconds
defp json_to_meta(json) do
raw = Jason.decode!(json)
%{
"bucket" => bucket,
"chunks" => chunks,
"digest" => digest,
"name" => name,
"nuid" => nuid,
"size" => size
} = raw
%Meta{
bucket: bucket,
chunks: chunks,
digest: digest,
deleted: Map.get(raw, "deleted", false),
name: name,
nuid: nuid,
size: size
}
end
defp purge_prior_chunks(conn, bucket, name) do
case info(conn, bucket, name) do
{:ok, meta} ->
Stream.purge(conn, stream_name(bucket), nil, %{filter: chunk_stream_topic(meta)})
{:error, %{"code" => 404}} ->
:ok
{:error, other} ->
{:error, other}
end
end
defp receive_all_metas(sid, num_pending, messages \\ [])
defp receive_all_metas(_sid, 0, messages) do
{:ok, messages}
end
defp receive_all_metas(sid, remaining, messages) do
receive do
{:msg, %{sid: ^sid, body: body}} ->
meta = json_to_meta(body)
receive_all_metas(sid, remaining - 1, [meta | messages])
after
10_000 ->
{:error, :timeout_waiting_for_messages}
end
end
defp receive_chunks(conn, %Meta{} = meta, chunk_fun) do
topic = chunk_stream_topic(meta)
stream = stream_name(meta.bucket)
inbox = Util.reply_inbox()
{:ok, sub} = Gnat.sub(conn, self(), inbox)
{:ok, consumer} =
Consumer.create(conn, %Consumer{
stream_name: stream,
deliver_subject: inbox,
deliver_policy: :all,
filter_subject: topic,
ack_policy: :none,
max_ack_pending: nil,
replay_policy: :instant,
max_deliver: 1,
flow_control: true,
idle_heartbeat: 5_000_000_000
})
:ok = receive_chunks(conn, sub, meta.chunks, chunk_fun)
:ok = Gnat.unsub(conn, sub)
:ok = Consumer.delete(conn, stream, consumer.name)
end
defp receive_chunks(_conn, _sub, 0, _chunk_fun) do
:ok
end
defp receive_chunks(conn, sub, remaining, chunk_fun) do
receive do
# Flow control message with reply - respond to it
{:msg, %{sid: ^sub, status: "100", description: "FlowControl Request", reply_to: reply}}
when not is_nil(reply) ->
Gnat.pub(conn, reply, "")
receive_chunks(conn, sub, remaining, chunk_fun)
# Flow control or heartbeat message without reply - get next message
{:msg, %{sid: ^sub, body: "", status: "100"}} ->
receive_chunks(conn, sub, remaining, chunk_fun)
# Regular data message
{:msg, %{sid: ^sub, body: body}} ->
chunk_fun.(body)
receive_chunks(conn, sub, remaining - 1, chunk_fun)
after
10_000 ->
{:error, :timeout_waiting_for_messages}
end
end
@chunk_size 128 * 1024
defp send_chunks(conn, io, topic) do
sha = :crypto.hash_init(:sha256)
size = 0
chunks = 0
send_chunks(conn, io, topic, sha, size, chunks)
end
defp send_chunks(conn, io, topic, sha, size, chunks) do
case IO.binread(io, @chunk_size) do
:eof ->
sha = :crypto.hash_final(sha)
{:ok, chunks, size, sha}
{:error, err} ->
{:error, err}
bytes ->
sha = :crypto.hash_update(sha, bytes)
size = size + byte_size(bytes)
chunks = chunks + 1
case Gnat.request(conn, topic, bytes) do
{:ok, _} ->
send_chunks(conn, io, topic, sha, size, chunks)
error ->
error
end
end
end
defp validate_bucket_name(name) do
case Regex.match?(~r/^[a-zA-Z0-9_-]+$/, name) do
true -> :ok
false -> {:error, "invalid bucket name"}
end
end
end
================================================
FILE: lib/gnat/jetstream/api/stream.ex
================================================
defmodule Gnat.Jetstream.API.Stream do
@moduledoc """
A module representing a NATS JetStream Stream.
Learn more about Streams: https://docs.nats.io/nats-concepts/jetstream/streams
## The Jetstream.API.Stream struct
The struct's mandatory fields are `:name` and `:subjects`. The rest will have the NATS
default values set.
Stream struct fields explanation:
* `:allow_direct` - Allow higher performance, direct access to get individual messages. E.g. KeyValue
* `:allow_msg_ttl` - Allow header initiated per-message TTLs.
* `:allow_rollup_hdrs` - allows the use of the Nats-Rollup header to replace all contents of a stream,
or subject in a stream, with a single new message.
* `:deny_delete` - restricts the ability to delete messages from a stream via the API. Cannot be changed
once set to true.
* `:deny_purge` - restricts the ability to purge messages from a stream via the API. Cannot be change
once set to true.
* `:description` - a short description of the purpose of this stream.
* `:discard` - determines what happens when a Stream reaches its limits. It has the following options:
- `:old` - the default option. Old messages are deleted.
- `:new` - refuses new messages.
* `:discard_new_per_subject` - - allows to enable discarding new messages per subject when limits are reached.
Requires `discard: :new` and the `:max_msgs_per_subject` to be configured.
* `:domain` - JetStream domain, mainly used for leaf nodes.
See [JetStream on Leaf Nodes](https://docs.nats.io/running-a-nats-service/configuration/leafnodes/jetstream_leafnodes).
* `:duplicate_window` - the window within which to track duplicate messages, expressed in nanoseconds.
* `:max_age` - maximum age of any message in the Stream, expressed in nanoseconds.
* `:max_bytes` - how many bytes the Stream may contain. Adheres to `:discard`, removing oldest or
refusing new messages if the Stream exceeds this size.
* `:max_consumers` - how many Consumers can be defined for a given Stream, -1 for unlimited.
* `:max_msg_size` - the largest message that will be accepted by the Stream.
* `:max_msgs_per_subject` - For wildcard streams ensure that for every unique subject this many messages are kept - a per subject retention limit.
Only available on nats-server versions greater than 2.3.0
* `:max_msgs` - how many messages may be in a Stream. Adheres to `:discard`, removing oldest or refusing
new messages if the Stream exceeds this number of messages
* `:mirror` - maintains a 1:1 mirror of another stream with name matching this property. When a mirror
is configured subjects and sources must be empty.
* `:mirror_direct` - Allow higher performance and unified direct access for mirrors as well.
* `:name` - a name for the Stream.
See [naming](https://docs.nats.io/running-a-nats-service/nats_admin/jetstream_admin/naming).
* `:no_ack` - disables acknowledging messages that are received by the Stream.
* `:num_replicas` - how many replicas to keep for each message.
* `:placement` - placement directives to consider when placing replicas of this stream, random placement
when unset. It has the following properties:
- `:cluster` - the desired cluster name to place the stream.
- `:tags` - tags required on servers hosting this stream.
* `:retention` - how messages are retained in the Stream. Once this is exceeded, old messages are removed.
It has the following options:
- `:limits` - the default policy.
- `:interest`
- `:workqueue`
* `:sealed` - sealed streams do not allow messages to be deleted via limits or API, sealed streams can not
be unsealed via configuration update. Can only be set on already created streams via the Update API.
* `:sources` - list of stream names to replicate into this stream.
* `:storage` - the type of storage backend. Available options:
- `:file`
- `:memory`
* `:compression` - If file-based and a compression algorithm is specified, the stream data will be compressed on disk.
Valid options are "none" for no compression or "s2" for Snappy compression.
* `:subjects` - a list of subjects to consume, supports wildcards.
* `:subject_delete_marker_ttl` - Enables and sets a duration expressed in nanoseconds. for adding server markers for
delete, purge and max age limits.
* `:template_owner` - when the Stream is managed by a Stream Template this identifies the template that
manages the Stream.
"""
import Gnat.Jetstream.API.Util
@enforce_keys [:name, :subjects]
@derive {Jason.Encoder, except: [:domain]}
defstruct [
:description,
:mirror,
:name,
:domain,
:no_ack,
:placement,
:sources,
:subjects,
:template_owner,
allow_direct: false,
allow_msg_ttl: false,
allow_rollup_hdrs: false,
deny_delete: false,
deny_purge: false,
discard: :old,
duplicate_window: 120_000_000_000,
max_age: 0,
max_bytes: -1,
max_consumers: -1,
max_msg_size: -1,
max_msgs_per_subject: -1,
max_msgs: -1,
mirror_direct: false,
num_replicas: 1,
retention: :limits,
sealed: false,
storage: :file,
subject_delete_marker_ttl: 0,
discard_new_per_subject: false,
compression: "none"
]
@type nanoseconds :: non_neg_integer()
@type placement :: %{
:cluster => binary(),
optional(:tags) => list(binary())
}
@type t :: %__MODULE__{
allow_direct: boolean(),
allow_msg_ttl: boolean(),
allow_rollup_hdrs: boolean(),
deny_delete: boolean(),
deny_purge: boolean(),
description: nil | binary(),
discard: :old | :new,
domain: nil | binary(),
duplicate_window: nil | nanoseconds(),
max_age: nanoseconds(),
max_bytes: integer(),
max_consumers: integer(),
max_msg_size: nil | integer(),
max_msgs: integer(),
max_msgs_per_subject: integer(),
mirror: nil | source(),
mirror_direct: boolean(),
name: binary(),
no_ack: nil | boolean(),
num_replicas: pos_integer(),
placement: nil | placement(),
retention: :limits | :workqueue | :interest,
sealed: boolean(),
sources: nil | list(source()),
storage: :file | :memory,
subjects: nil | list(binary()),
subject_delete_marker_ttl: nanoseconds(),
template_owner: nil | binary(),
discard_new_per_subject: boolean(),
compression: binary()
}
@typedoc """
Stream source fields explained:
* `:name` - stream name.
* `:opt_start_seq` - sequence to start replicating from.
* `:opt_start_time` - timestamp to start replicating from.
* `:filter_subject` - replicate only a subset of messages based on filter.
* `:external` - configuration referencing a stream source in another account or JetStream domain.
It has the following parameters:
- `:api` - the subject prefix that imports other account/domain `$JS.API.CONSUMER.>` subjects
- `:deliver` - the delivery subject to use for push consumer
"""
@type source :: %{
:name => binary(),
optional(:opt_start_seq) => integer(),
optional(:opt_start_time) => DateTime.t(),
optional(:filter_subject) => binary(),
optional(:external) => %{
api: binary(),
deliver: binary()
}
}
@type info :: %{
cluster:
nil
| %{
optional(:name) => binary(),
optional(:leader) => binary(),
optional(:replicas) =>
list(%{
:active => nanoseconds(),
:name => binary(),
:current => boolean(),
optional(:offline) => boolean(),
optional(:lag) => non_neg_integer()
})
},
config: t(),
created: DateTime.t(),
mirror: nil | source_info(),
sources: nil | list(source_info()),
state: state()
}
@type state :: %{
bytes: non_neg_integer(),
consumer_count: non_neg_integer(),
deleted: nil | [non_neg_integer()],
first_seq: non_neg_integer(),
first_ts: DateTime.t(),
last_seq: non_neg_integer(),
last_ts: DateTime.t(),
lost: nil | list(%{msgs: [non_neg_integer()], bytes: non_neg_integer()}),
messages: non_neg_integer(),
num_deleted: nil | integer(),
num_subjects: nil | integer(),
subjects:
nil
| %{}
| %{
binary() => non_neg_integer()
}
}
@typedoc """
* `code` - HTTP like error code in the 300 to 500 range
* `description` - A human friendly description of the error
* `err_code` - The NATS error code unique to each kind of error
"""
@type response_error :: %{
:code => non_neg_integer(),
optional(:description) => binary(),
optional(:err_code) => non_neg_integer()
}
@type source_info :: %{
:active => nanoseconds(),
:lag => non_neg_integer(),
:name => binary(),
optional(:external) => %{
api: binary(),
deliver: binary()
},
optional(:error) => response_error()
}
@type streams :: %{
limit: non_neg_integer(),
offset: non_neg_integer(),
streams: list(binary()),
total: non_neg_integer()
}
@typedoc """
* `seq` - Stream sequence number of the message to retrieve, cannot be combined with `last_by_subj`
* `last_by_subj` - Retrieves the last message for a given subject, cannot be combined with `seq`
"""
@type message_access_method :: %{
optional(:seq) => non_neg_integer(),
optional(:last_by_subj) => binary()
}
@typedoc """
* `data` - The decoded message payload
* `subject` - The subject the message was originally received on
* `time` - The time the message was received
* `seq` - The sequence number of the message in the Stream
* `hdrs` - The decoded headers for the message
"""
@type message_response :: %{
:data => any(),
:seq => non_neg_integer(),
:subject => binary(),
:time => DateTime.t(),
:hdrs => nil | binary()
}
# @doc """
# Initialize a Stream struct
# ## Examples
# iex> %Stream{} = Gnat.Jetstream.API.Stream.new(:gnat, name: "NEW_STREAM", subjects: ["NEW_STREAM.subjects"])
# """
# @spec new(conn :: Gnat.t(), fields :: keyword()) :: t()
# def new(conn, fields \\ []) do
# %__MODULE__{}
# end
@doc """
Creates a new Stream.
## Examples
iex> {:ok, %{created: _}} = Gnat.Jetstream.API.Stream.create(:gnat, %Gnat.Jetstream.API.Stream{name: "anewstream", subjects: ["anewsubject"]})
"""
@spec create(conn :: Gnat.t(), stream :: t()) :: {:ok, info()} | {:error, any()}
def create(conn, %__MODULE__{} = stream) do
with :ok <- validate(stream),
{:ok, stream} <-
request(
conn,
"#{js_api(stream.domain)}.STREAM.CREATE.#{stream.name}",
Jason.encode!(stream)
) do
{:ok, to_info(stream)}
end
end
@doc """
Updates a Stream.
## Examples
iex> {:ok, %{created: _}} = Gnat.Jetstream.API.Stream.create(:gnat, %Gnat.Jetstream.API.Stream{name: "update_test_stream", subjects: ["update_subject"]})
iex> {:ok, _} = Gnat.Jetstream.API.Stream.update(:gnat, %Gnat.Jetstream.API.Stream{name: "update_test_stream", subjects: ["update_subject", "new.update_subject"]})
"""
@spec update(conn :: Gnat.t(), stream :: t()) :: {:ok, info()} | {:error, any()}
def update(conn, %__MODULE__{} = stream) do
with :ok <- validate(stream),
{:ok, stream} <-
request(
conn,
"#{js_api(stream.domain)}.STREAM.UPDATE.#{stream.name}",
Jason.encode!(stream)
) do
{:ok, to_info(stream)}
end
end
@doc """
Deletes a Stream and all its data.
## Examples
iex> Gnat.Jetstream.API.Stream.create(:gnat, %Gnat.Jetstream.API.Stream{name: "delstream", subjects: ["delsubject"]})
iex> Gnat.Jetstream.API.Stream.delete(:gnat, "delstream")
:ok
iex> {:error, %{"code" => 404, "description" => "stream not found"}} = Gnat.Jetstream.API.Stream.delete(:gnat, "wrong_stream")
"""
@spec delete(conn :: Gnat.t(), stream_name :: binary(), domain :: nil | binary()) ::
:ok | {:error, any()}
def delete(conn, stream_name, domain \\ nil) when is_binary(stream_name) do
with {:ok, _response} <- request(conn, "#{js_api(domain)}.STREAM.DELETE.#{stream_name}", "") do
:ok
end
end
@doc """
Purges all of data in the stream but doesn't delete the stream.
## Examples
iex> Gnat.Jetstream.API.Stream.create(:gnat, %Gnat.Jetstream.API.Stream{name: "purgestream", subjects: ["purgesubject"]})
iex> Gnat.Jetstream.API.Stream.purge(:gnat, "purgestream")
:ok
iex> {:error, %{"code" => 404, "description" => "stream not found"}} = Gnat.Jetstream.API.Stream.purge(:gnat, "wrong_stream")
"""
@spec purge(conn :: Gnat.t(), stream_name :: binary(), domain :: nil | binary()) ::
:ok | {:error, any()}
def purge(conn, stream_name, domain \\ nil) when is_binary(stream_name) do
with {:ok, _response} <- request(conn, "#{js_api(domain)}.STREAM.PURGE.#{stream_name}", "") do
:ok
end
end
@doc """
Purges some of the messages in a stream according to the supplied filter
## Examples
iex> Gnat.Jetstream.API.Stream.create(:gnat, %Gnat.Jetstream.API.Stream{name: "pstream", subjects: ["psub1", "psub2"]})
iex> Gnat.Jetstream.API.Stream.purge(:gnat, "pstream", nil, %{filter: "psub1"})
:ok
"""
@type method :: %{filter: String.t()}
@spec purge(conn :: Gnat.t(), stream_name :: binary(), domain :: nil | binary(), method) ::
:ok | {:error, any()}
def purge(conn, stream_name, domain, method) when is_binary(stream_name) do
with :ok <- validate_purge_method(method),
body <- Jason.encode!(method),
{:ok, _response} <- request(conn, "#{js_api(domain)}.STREAM.PURGE.#{stream_name}", body) do
:ok
end
end
@doc """
Information about config and state of a Stream.
## Examples
iex> {:ok, _} = Gnat.Jetstream.API.Stream.create(:gnat, %Gnat.Jetstream.API.Stream{name: "infostream", subjects: ["infosubject"]})
iex> {:ok, %{created: _}} = Gnat.Jetstream.API.Stream.info(:gnat, "infostream")
iex> {:error, %{"code" => 404, "description" => "stream not found"}} = Gnat.Jetstream.API.Stream.info(:gnat, "wrong_stream")
"""
@spec info(conn :: Gnat.t(), stream_name :: binary(), domain :: nil | binary()) ::
{:ok, info()} | {:error, any()}
def info(conn, stream_name, domain \\ nil) when is_binary(stream_name) do
with {:ok, decoded} <- request(conn, "#{js_api(domain)}.STREAM.INFO.#{stream_name}", "") do
{:ok, to_info(decoded)}
end
end
@doc """
Paged list of known Streams including all their current information.
## Options
* `:offset` - Number of records to skip
* `:subject` - A subject the `Stream` must collect to appear in the list.
## Examples
iex> {:ok, %{total: _, offset: 0, limit: 1024, streams: _}} = Gnat.Jetstream.API.Stream.list(:gnat)
"""
@spec list(
conn :: Gnat.t(),
params :: [offset: non_neg_integer(), subject: binary(), domain: nil | binary()]
) ::
{:ok, streams()} | {:error, term()}
def list(conn, params \\ []) do
domain = Keyword.get(params, :domain)
payload =
Jason.encode!(%{
offset: Keyword.get(params, :offset, 0),
subject: Keyword.get(params, :subject)
})
with {:ok, decoded} <- request(conn, "#{js_api(domain)}.STREAM.NAMES", payload) do
# Recent versions of NATS sometimes return `"streams": null` in their JSON payload to indicate
# that no streams are defined. But, that would mean callers have to handle both `nil` and a list, so
# we coerce that to an empty list to represent no streams being defined.
streams =
case Map.get(decoded, "streams") do
nil -> []
names when is_list(names) -> names
end
result = %{
limit: Map.get(decoded, "limit"),
offset: Map.get(decoded, "offset"),
streams: streams,
total: Map.get(decoded, "total")
}
{:ok, result}
end
end
@doc """
Get a message from the stream either by "stream sequence number" or the "last message for a given subject"
"""
@spec get_message(
conn :: Gnat.t(),
stream_name :: binary(),
method :: message_access_method(),
domain :: nil | binary()
) ::
{:ok, message_response()} | {:error, response_error()}
def get_message(conn, stream_name, method, domain \\ nil) when is_map(method) do
with :ok <- validate_message_access_method(method),
{:ok, %{"message" => message}} <-
request(conn, "#{js_api(domain)}.STREAM.MSG.GET.#{stream_name}", Jason.encode!(method)) do
{:ok,
%{
data: decode_base64(message["data"]),
seq: message["seq"],
subject: message["subject"],
time: to_datetime(message["time"]),
hdrs: decode_base64(message["hdrs"])
}}
end
end
# https://docs.nats.io/running-a-nats-service/configuration/leafnodes/jetstream_leafnodes
defp js_api(nil), do: "$JS.API"
defp js_api(""), do: "$JS.API"
defp js_api(domain), do: "$JS.#{domain}.API"
defp to_state(state) do
%{
bytes: Map.fetch!(state, "bytes"),
consumer_count: Map.fetch!(state, "consumer_count"),
deleted: Map.get(state, "deleted"),
first_seq: Map.fetch!(state, "first_seq"),
first_ts: Map.fetch!(state, "first_ts") |> to_datetime(),
last_seq: Map.fetch!(state, "last_seq"),
last_ts: Map.fetch!(state, "last_ts") |> to_datetime(),
lost: Map.get(state, "lost"),
messages: Map.fetch!(state, "messages"),
num_deleted: Map.get(state, "num_deleted"),
num_subjects: Map.get(state, "num_subjects"),
subjects: Map.get(state, "subjects")
}
end
defp to_stream(stream) do
%__MODULE__{
description: Map.get(stream, "description"),
discard: Map.fetch!(stream, "discard") |> to_sym(),
duplicate_window: Map.get(stream, "duplicate_window"),
max_age: Map.fetch!(stream, "max_age"),
max_bytes: Map.fetch!(stream, "max_bytes"),
max_consumers: Map.fetch!(stream, "max_consumers"),
max_msg_size: Map.get(stream, "max_msg_size"),
max_msgs_per_subject: Map.get(stream, "max_msgs_per_subject", -1),
max_msgs: Map.fetch!(stream, "max_msgs"),
mirror: Map.get(stream, "mirror"),
name: Map.fetch!(stream, "name"),
no_ack: Map.get(stream, "no_ack"),
num_replicas: Map.fetch!(stream, "num_replicas"),
placement: Map.get(stream, "placement"),
retention: Map.fetch!(stream, "retention") |> to_sym(),
sources: Map.get(stream, "sources"),
storage: Map.fetch!(stream, "storage") |> to_sym(),
subjects: Map.get(stream, "subjects"),
template_owner: Map.get(stream, "template_owner")
}
# Check for fields added in NATS versions higher than 2.2.0
|> put_if_exist(:allow_direct, stream, "allow_direct")
|> put_if_exist(:allow_msg_ttl, stream, "allow_msg_ttl")
|> put_if_exist(:allow_rollup_hdrs, stream, "allow_rollup_hdrs")
|> put_if_exist(:deny_delete, stream, "deny_delete")
|> put_if_exist(:deny_purge, stream, "deny_purge")
|> put_if_exist(:discard_new_per_subject, stream, "discard_new_per_subject")
|> put_if_exist(:mirror_direct, stream, "mirror_direct")
|> put_if_exist(:sealed, stream, "sealed")
|> put_if_exist(:subject_delete_marker_ttl, stream, "subject_delete_marker_ttl")
|> put_if_exist(:compression, stream, "compression")
end
defp to_info(%{"config" => config, "state" => state, "created" => created} = response) do
with {:ok, created, _} <- DateTime.from_iso8601(created) do
%{
cluster: Map.get(response, "cluster"),
config: to_stream(config),
created: created,
mirror: Map.get(response, "mirror"),
sources: Map.get(response, "sources"),
state: to_state(state)
}
end
end
defp validate(stream_settings) do
cond do
Map.has_key?(stream_settings, :name) == false ->
{:error, "Must have a :name set"}
is_binary(Map.get(stream_settings, :name)) == false ->
{:error, "name must be a string"}
valid_name?(stream_settings.name) == false ->
{:error, "invalid name: " <> invalid_name_message()}
Map.has_key?(stream_settings, :subjects) == false ->
{:error, "You must specify a :subjects key"}
is_list(Map.get(stream_settings, :subjects)) == false ->
{:error, ":subjects must be a list of strings"}
Enum.all?(Map.get(stream_settings, :subjects), &is_binary/1) == false ->
{:error, ":subjects must be a list of strings"}
true ->
:ok
end
end
defp validate_message_access_method(method) do
if map_size(method) == 1 do
:ok
else
{:error, "To get a message you must use only one of `seq` or `last_by_subj`"}
end
end
defp validate_purge_method(%{filter: subject}) when is_binary(subject) do
:ok
end
defp validate_purge_method(_) do
{:error, "When purging, you must pass a %{filter: subject}"}
end
end
================================================
FILE: lib/gnat/jetstream/api/util.ex
================================================
defmodule Gnat.Jetstream.API.Util do
@moduledoc false
@default_inbox_prefix "_INBOX."
def request(conn, topic, payload) do
with {:ok, %{body: body}} <- Gnat.request(conn, topic, payload),
{:ok, decoded} <- Jason.decode(body) do
case decoded do
%{"error" => err} ->
{:error, err}
other ->
{:ok, other}
end
end
end
def to_datetime(nil), do: nil
def to_datetime(str) do
{:ok, datetime, _} = DateTime.from_iso8601(str)
datetime
end
def to_sym(nil), do: nil
def to_sym(str) when is_binary(str) do
String.to_existing_atom(str)
end
def put_if_exist(target_map, target_key, source_map, source_key) do
case Map.fetch(source_map, source_key) do
{:ok, value} -> Map.put(target_map, target_key, value)
_ -> target_map
end
end
def valid_name?(name) do
!String.contains?(name, [".", "*", ">", " ", "\t"])
end
def invalid_name_message do
"cannot contain '.', '>', '*', spaces or tabs"
end
def decode_base64(nil), do: nil
def decode_base64(data), do: Base.decode64!(data)
def reply_inbox(prefix \\ @default_inbox_prefix)
def reply_inbox(nil), do: reply_inbox()
def reply_inbox(prefix), do: prefix <> nuid()
def nuid() do
:crypto.strong_rand_bytes(12) |> Base.url_encode64()
end
end
================================================
FILE: lib/gnat/jetstream/jetstream.ex
================================================
defmodule Gnat.Jetstream do
@moduledoc """
Provides functions for interacting with a [NATS Jetstream](https://github.com/nats-io/jetstream)
server.
"""
@doc """
Sends `AckAck` acknowledgement to the server.
Acknowledges a message was completely handled.
"""
@spec ack(message :: Gnat.message()) :: :ok
def ack(%{reply_to: nil}) do
{:error, "Cannot ack message with no reply-to"}
end
def ack(%{gnat: gnat, reply_to: reply_to}) do
Gnat.pub(gnat, reply_to, "")
end
@doc """
Sends `AckNext` acknowledgement to the server.
Acknowledges the message was handled and requests delivery of the next message to the reply
subject. Only applies to Pull-mode.
"""
@spec ack_next(message :: Gnat.message(), consumer_subject :: binary()) :: :ok
def ack_next(%{reply_to: nil}, _consumer_subject) do
{:error, "Cannot ack message with no reply-to"}
end
def ack_next(%{gnat: gnat, reply_to: reply_to}, consumer_subject) do
Gnat.pub(gnat, reply_to, "+NXT", reply_to: consumer_subject)
end
@doc """
Sends `AckNak` acknowledgement to the server.
Signals that the message will not be processed now and processing can move onto the next message.
NAK'd message will be retried.
"""
@spec nack(message :: Gnat.message()) :: :ok
def nack(%{reply_to: nil}) do
{:error, "Cannot ack message with no reply-to"}
end
def nack(%{gnat: gnat, reply_to: reply_to}) do
Gnat.pub(gnat, reply_to, "-NAK")
end
@doc """
Sends `AckProgress` acknowledgement to the server.
When sent before the `AckWait` period indicates that work is ongoing and the period should be
extended by another equal to `AckWait`.
"""
@spec ack_progress(message :: Gnat.message()) :: :ok
def ack_progress(%{reply_to: nil}) do
{:error, "Cannot ack message with no reply-to"}
end
def ack_progress(%{gnat: gnat, reply_to: reply_to}) do
Gnat.pub(gnat, reply_to, "+WPI")
end
@doc """
Sends `AckTerm` acknowledgement to the server.
Instructs the server to stop redelivery of a message without acknowledging it as successfully
processed.
"""
@spec ack_term(message :: Gnat.message()) :: :ok
def ack_term(%{reply_to: nil}) do
{:error, "Cannot ack message with no reply-to"}
end
def ack_term(%{gnat: gnat, reply_to: reply_to}) do
Gnat.pub(gnat, reply_to, "+TERM")
end
end
================================================
FILE: lib/gnat/jetstream/pager.ex
================================================
defmodule Gnat.Jetstream.Pager do
@moduledoc """
Page through all the messages in a stream
This module provides a synchronous API to inspect the messages in a stream.
You can use the reduce module to write a simple function that works like `Enum.reduce` across each message individually.
If you want to handle messages in batches, you can use the `init` + `page` functions to accomplish that.
"""
alias Gnat.Jetstream
alias Gnat.Jetstream.API.{Consumer, Util}
@opaque pager :: map()
@type message :: Gnat.message()
@type opts :: list(opt())
@typedoc """
Options you can pass to the pager
* `batch` controls the maximum number of messages we'll pull in each page/batch (default 10)
* `domain` You can specify a jetstream domain if needed
* `from_datetime` Only page through messages recorded on or after this datetime
* `from_seq` Only page through messages with a sequence number equal or above this option
* `headers_only` You can pass `true` to this if you only want to see the headers from each message. Can be useful to get metadata without having to receieve large body payloads.
"""
@type opt ::
{:batch, non_neg_integer()}
| {:domain, String.t()}
| {:from_datetime, DateTime.t()}
| {:from_seq, non_neg_integer}
| {:headers_only, boolean()}
@spec init(Gnat.t(), String.t(), opts()) :: {:ok, pager()} | {:error, term()}
def init(conn, stream_name, opts) do
domain = Keyword.get(opts, :domain)
consumer = %Consumer{
stream_name: stream_name,
domain: domain,
ack_policy: :all,
ack_wait: 30_000_000_000,
deliver_policy: :all,
description: "ephemeral consumer",
replay_policy: :instant,
inactive_threshold: 30_000_000_000,
headers_only: Keyword.get(opts, :headers_only)
}
consumer = apply_opts_to_consumer(consumer, opts)
inbox = Util.reply_inbox()
with {:ok, consumer_info} <- Consumer.create(conn, consumer),
{:ok, sub} <- Gnat.sub(conn, self(), inbox) do
state = %{
conn: conn,
stream_name: stream_name,
consumer_name: consumer_info.name,
domain: domain,
inbox: inbox,
batch: Keyword.get(opts, :batch, 10),
sub: sub
}
{:ok, state}
end
end
@spec page(pager()) :: {:page, list(message())} | {:done, list(message())} | {:error, term()}
def page(state) do
with :ok <- request_next_message(state) do
receive_messages(state, [])
end
end
def cleanup(%{conn: conn} = state) do
:ok = Gnat.unsub(conn, state.sub)
Consumer.delete(conn, state.stream_name, state.consumer_name, state.domain)
end
@doc """
Similar to Enum.reduce but you can iterate through all messages in a stream
```
# Assume we have a stream with messages like "1", "2", ... "10"
Gnat.Jetstream.Pager.reduce(:gnat, "NUMBERS_STREAM", [batch_size: 5], 0, fn(message, total) ->
num = String.to_integer(message.body)
total + num
end)
# => {:ok, 55}
```
"""
@spec reduce(
Gnat.t(),
String.t(),
opts(),
Enum.acc(),
(Gnat.message(), Enum.acc() -> Enum.acc())
) :: {:ok, Enum.acc()} | {:error, term()}
def reduce(conn, stream_name, opts, initial_state, fun) do
with {:ok, pager} <- init(conn, stream_name, opts) do
page_through(pager, initial_state, fun)
end
end
defp page_through(pager, state, fun) do
case page(pager) do
{:page, messages} ->
new_state = Enum.reduce(messages, state, fun)
page_through(pager, new_state, fun)
{:done, messages} ->
new_state = Enum.reduce(messages, state, fun)
cleanup(pager)
{:ok, new_state}
end
end
defp request_next_message(state) do
opts = [batch: state.batch, no_wait: true]
Consumer.request_next_message(
state.conn,
state.stream_name,
state.consumer_name,
state.inbox,
state.domain,
opts
)
end
defp receive_messages(%{batch: batch}, messages) when length(messages) == batch do
last = hd(messages)
:ok = Jetstream.ack(last)
{:page, Enum.reverse(messages)}
end
@terminals ["404", "408"]
defp receive_messages(%{sub: sid} = state, messages) do
receive do
{:msg, %{sid: ^sid, status: status}} when status in @terminals ->
{:done, Enum.reverse(messages)}
{:msg, %{sid: ^sid, reply_to: nil}} ->
{:done, Enum.reverse(messages)}
{:msg, %{sid: ^sid} = message} ->
receive_messages(state, [message | messages])
end
end
## Helpers for accepting user options
defp apply_opts_to_consumer(consumer = %Consumer{}, opts) do
from = {Keyword.get(opts, :from_seq), Keyword.get(opts, :from_datetime)}
case from do
{nil, nil} ->
consumer
{seq, _} when is_integer(seq) ->
%Consumer{consumer | deliver_policy: :by_start_sequence, opt_start_seq: seq}
{_, %DateTime{} = dt} ->
%Consumer{consumer | deliver_policy: :by_start_time, opt_start_time: dt}
end
end
end
================================================
FILE: lib/gnat/jetstream/pull_consumer/connection_options.ex
================================================
defmodule Gnat.Jetstream.PullConsumer.ConnectionOptions do
@moduledoc false
@default_retry_timeout 1000
@default_retries 10
@enforce_keys [
:connection_name,
:connection_retry_timeout,
:connection_retries,
:inbox_prefix,
:domain
]
# 5 Seconds in nanoseconds
@default_request_expires 5_000_000_000
defstruct @enforce_keys ++
[
:stream_name,
:consumer_name,
:consumer,
batch_size: 1,
request_expires: @default_request_expires
]
def validate!(connection_options) do
validated_opts =
Keyword.validate!(connection_options, [
:connection_name,
:stream_name,
:consumer_name,
:consumer,
connection_retry_timeout: @default_retry_timeout,
connection_retries: @default_retries,
inbox_prefix: nil,
domain: nil,
batch_size: 1,
request_expires: @default_request_expires
])
stream_name = validated_opts[:stream_name]
consumer_name = validated_opts[:consumer_name]
consumer = validated_opts[:consumer]
cond do
consumer && (stream_name || consumer_name) ->
raise ArgumentError,
"cannot specify :consumer with :stream_name or :consumer_name - use consumer struct's stream_name instead"
consumer && !is_struct(consumer, Gnat.Jetstream.API.Consumer) ->
raise ArgumentError, ":consumer must be a Consumer struct"
consumer && consumer.durable_name != nil && consumer.inactive_threshold == nil ->
raise ArgumentError,
"durable consumers specified via :consumer must have inactive_threshold set for auto-cleanup"
consumer && validated_opts[:batch_size] > 1 && consumer.ack_policy != :all ->
raise ArgumentError,
"batch_size > 1 requires ack_policy: :all on the consumer, " <>
"got: #{inspect(consumer.ack_policy)}. With ack_policy: :explicit, " <>
"only the last message in each batch would be acknowledged and the " <>
"server would redeliver the rest"
consumer ->
# For ephemeral/auto-cleanup consumer case, extract stream_name from consumer struct
validated_opts = Keyword.put(validated_opts, :stream_name, consumer.stream_name)
struct!(__MODULE__, validated_opts)
stream_name && consumer_name ->
# For traditional durable consumer case
struct!(__MODULE__, validated_opts)
true ->
raise ArgumentError,
"must specify either :consumer (ephemeral/auto-cleanup) or both :stream_name and :consumer_name (durable)"
end
end
end
================================================
FILE: lib/gnat/jetstream/pull_consumer/server.ex
================================================
defmodule Gnat.Jetstream.PullConsumer.Server do
@moduledoc false
require Logger
use Connection
alias Gnat.Jetstream.PullConsumer.ConnectionOptions
alias Gnat.Jetstream.API.Util
defstruct [
:connection_options,
:state,
:listening_topic,
:module,
:subscription_id,
:connection_monitor_ref,
:consumer_name,
current_retry: 0,
buffer: []
]
def init(%{module: module, init_arg: init_arg}) do
_ = Process.put(:"$initial_call", {module, :init, 1})
case module.init(init_arg) do
{:ok, state, connection_options} when is_list(connection_options) ->
Process.flag(:trap_exit, true)
connection_options = ConnectionOptions.validate!(connection_options)
gen_state = %__MODULE__{
connection_options: connection_options,
state: state,
listening_topic: Util.reply_inbox(connection_options.inbox_prefix),
module: module,
consumer_name: connection_options.consumer_name
}
{:connect, :init, gen_state}
:ignore ->
:ignore
{:stop, _} = stop ->
stop
end
end
def connect(
_,
%__MODULE__{
connection_options: %ConnectionOptions{
stream_name: stream_name,
consumer_name: consumer_name,
consumer: consumer,
connection_name: connection_name,
connection_retry_timeout: connection_retry_timeout,
connection_retries: connection_retries,
domain: domain
},
listening_topic: listening_topic,
module: module
} = gen_state
) do
Logger.debug(
"#{__MODULE__} for #{stream_name}.#{gen_state.consumer_name} is connecting to Gnat.",
module: module,
listening_topic: listening_topic,
connection_name: connection_name
)
with {:ok, conn} <- connection_pid(connection_name),
monitor_ref = Process.monitor(conn),
{:ok, consumer_info} <-
ensure_consumer_exists(
conn,
stream_name,
consumer_name,
consumer,
domain
),
:ok <- validate_batch_ack_policy(gen_state.connection_options, consumer_info),
final_consumer_name = consumer_info.name,
state = maybe_handle_connected(module, consumer_info, gen_state.state),
{:ok, sid} <- Gnat.sub(conn, self(), listening_topic),
gen_state = %{
gen_state
| subscription_id: sid,
connection_monitor_ref: monitor_ref,
consumer_name: final_consumer_name,
state: state
},
:ok <-
initial_fetch(
gen_state,
conn,
stream_name,
final_consumer_name,
domain,
listening_topic
),
gen_state = %{gen_state | current_retry: 0} do
{:ok, gen_state}
else
{:error, reason} ->
if gen_state.current_retry >= connection_retries do
Logger.error(
"""
#{__MODULE__} for #{stream_name}.#{gen_state.consumer_name} failed to connect to NATS and \
retries limit has been exhausted. Stopping.
""",
module: module,
listening_topic: listening_topic,
connection_name: connection_name
)
{:stop, :timeout, %{gen_state | current_retry: 0}}
else
Logger.debug(
"""
#{__MODULE__} for #{stream_name}.#{gen_state.consumer_name} failed to connect to Gnat \
and will retry. Reason: #{inspect(reason)}
""",
module: module,
listening_topic: listening_topic,
connection_name: connection_name
)
gen_state = Map.update!(gen_state, :current_retry, &(&1 + 1))
{:backoff, connection_retry_timeout, gen_state}
end
end
end
def disconnect(
{:close, from},
%__MODULE__{
connection_options: %ConnectionOptions{
stream_name: stream_name,
connection_name: connection_name
},
listening_topic: listening_topic,
subscription_id: subscription_id,
module: module,
consumer_name: consumer_name
} = gen_state
) do
Logger.debug(
"#{__MODULE__} for #{stream_name}.#{consumer_name} is disconnecting from Gnat.",
module: module,
listening_topic: listening_topic,
subscription_id: subscription_id,
connection_name: connection_name
)
with {:ok, conn} <- connection_pid(connection_name),
true <- Process.demonitor(gen_state.connection_monitor_ref, [:flush]),
:ok <- Gnat.unsub(conn, subscription_id) do
Logger.debug(
"#{__MODULE__} for #{stream_name}.#{consumer_name} is shutting down.",
module: module,
listening_topic: listening_topic,
subscription_id: subscription_id,
connection_name: connection_name
)
Connection.reply(from, :ok)
{:stop, :shutdown, gen_state}
end
end
defp ensure_consumer_exists(gnat, stream_name, consumer_name, nil, domain) do
# Durable consumer case - just check it exists
try do
case Gnat.Jetstream.API.Consumer.info(gnat, stream_name, consumer_name, domain) do
{:ok, consumer_info} -> {:ok, consumer_info}
{:error, reason} -> {:error, reason}
end
catch
:exit, reason -> {:error, {:process_exit, reason}}
kind, reason -> {:error, {kind, reason}}
end
end
defp ensure_consumer_exists(gnat, _stream_name, nil, consumer_struct, _domain) do
# Ephemeral or auto-cleanup durable consumer case - create it
try do
with {:ok, consumer_definition} <- validate_consumer_for_creation(consumer_struct),
{:ok, consumer_info} <- Gnat.Jetstream.API.Consumer.create(gnat, consumer_definition) do
{:ok, consumer_info}
end
catch
:exit, reason -> {:error, {:process_exit, reason}}
kind, reason -> {:error, {kind, reason}}
end
end
defp validate_consumer_for_creation(consumer_definition) do
cond do
consumer_definition.durable_name == nil && consumer_definition.inactive_threshold != nil ->
{:error, "ephemeral consumers (durable_name: nil) cannot have inactive_threshold set"}
consumer_definition.durable_name != nil && consumer_definition.inactive_threshold == nil ->
{:error,
"durable consumers specified via :consumer must have inactive_threshold set for auto-cleanup"}
true ->
{:ok, consumer_definition}
end
end
defp validate_batch_ack_policy(%ConnectionOptions{batch_size: batch_size}, consumer_info)
when batch_size > 1 do
case consumer_info.config.ack_policy do
:all ->
:ok
other ->
{:error,
"batch_size > 1 requires ack_policy: :all on the consumer, " <>
"got: #{inspect(other)}. With ack_policy: :explicit, " <>
"only the last message in each batch would be acknowledged and the " <>
"server would redeliver the rest"}
end
end
defp validate_batch_ack_policy(_connection_options, _consumer_info), do: :ok
defp maybe_handle_connected(module, consumer_info, state) do
if function_exported?(module, :handle_connected, 2) do
{:ok, state} = module.handle_connected(consumer_info, state)
state
else
state
end
end
defp maybe_handle_status(message, %__MODULE__{module: module, state: state} = gen_state) do
if function_exported?(module, :handle_status, 2) do
{:ok, new_state} = module.handle_status(message, state)
%{gen_state | state: new_state}
else
gen_state
end
end
defp connection_pid(connection_name) when is_pid(connection_name) do
if Process.alive?(connection_name) do
{:ok, connection_name}
else
{:error, :not_alive}
end
end
defp connection_pid(connection_name) do
case Process.whereis(connection_name) do
nil -> {:error, :not_found}
pid -> {:ok, pid}
end
end
# -- Batch mode: 100 is an idle heartbeat — the pull is still alive, do
# nothing but invoke the user callback. --
def handle_info(
{:msg, %{status: "100"} = message},
%__MODULE__{connection_options: %ConnectionOptions{batch_size: batch_size}} = gen_state
)
when batch_size > 1 do
gen_state = maybe_handle_status(message, gen_state)
{:noreply, gen_state}
end
# -- Batch mode: any other status (404/408 terminators, 409 leadership
# change / max_ack_pending / max_waiting / consumer-deleted, etc.) ends
# the outstanding pull request. Process any partial buffer and issue a
# new pull so the consumer doesn't stall. --
def handle_info(
{:msg, %{status: status, gnat: gnat} = message},
%__MODULE__{
connection_options: %ConnectionOptions{batch_size: batch_size},
buffer: buffer
} = gen_state
)
when batch_size > 1 and is_binary(status) and status != "" do
gen_state = maybe_handle_status(message, gen_state)
case buffer do
[] ->
# Nothing buffered — long-poll for new messages.
request_batch(gnat, gen_state, :tailing)
{:noreply, gen_state}
_messages ->
# Partial batch — process what we have, then try for more.
gen_state = process_and_ack_batch(gen_state)
request_batch(gnat, gen_state, :catching_up)
{:noreply, gen_state}
end
end
# -- Single-message mode: informational status. Drop + re-pull so the
# consumer doesn't stall. Matches the nats.go convention of never exposing
# status messages to the user's message handler. --
def handle_info(
{:msg, %{status: status} = message},
%__MODULE__{
connection_options: %ConnectionOptions{
stream_name: stream_name,
domain: domain
},
listening_topic: listening_topic,
consumer_name: consumer_name
} = gen_state
)
when is_binary(status) and status != "" do
gen_state = maybe_handle_status(message, gen_state)
next_message(
message.gnat,
stream_name,
consumer_name,
domain,
listening_topic
)
{:noreply, gen_state}
end
# -- Batch mode: data message — buffer until batch is full --
def handle_info(
{:msg, message},
%__MODULE__{
connection_options: %ConnectionOptions{batch_size: batch_size},
buffer: buffer
} = gen_state
)
when batch_size > 1 do
buffer = [message | buffer]
gen_state = %{gen_state | buffer: buffer}
if length(buffer) >= batch_size do
gen_state = process_and_ack_batch(gen_state)
request_batch(message.gnat, gen_state, :catching_up)
{:noreply, gen_state}
else
{:noreply, gen_state}
end
end
# -- Single-message mode (batch_size == 1, the default) --
def handle_info(
{:msg, message},
%__MODULE__{
connection_options: %ConnectionOptions{
stream_name: stream_name,
connection_name: connection_name,
domain: domain
},
listening_topic: listening_topic,
subscription_id: subscription_id,
state: state,
module: module,
consumer_name: consumer_name
} = gen_state
) do
Logger.debug(
"""
#{__MODULE__} for #{stream_name}.#{consumer_name} received a message: \
#{inspect(message, pretty: true)}
""",
module: module,
listening_topic: listening_topic,
subscription_id: subscription_id,
connection_name: connection_name
)
case module.handle_message(message, state) do
{:ack, state} ->
Gnat.Jetstream.ack_next(message, listening_topic)
gen_state = %{gen_state | state: state}
{:noreply, gen_state}
{:nack, state} ->
Gnat.Jetstream.nack(message)
next_message(
message.gnat,
stream_name,
consumer_name,
domain,
listening_topic
)
gen_state = %{gen_state | state: state}
{:noreply, gen_state}
{:term, state} ->
Gnat.Jetstream.ack_term(message)
next_message(
message.gnat,
stream_name,
consumer_name,
domain,
listening_topic
)
gen_state = %{gen_state | state: state}
{:noreply, gen_state}
{:noreply, state} ->
next_message(
message.gnat,
stream_name,
consumer_name,
domain,
listening_topic
)
gen_state = %{gen_state | state: state}
{:noreply, gen_state}
end
end
def handle_info(
{:DOWN, ref, :process, _pid, _reason},
%__MODULE__{
connection_options: %ConnectionOptions{
connection_name: connection_name,
stream_name: stream_name
},
subscription_id: subscription_id,
listening_topic: listening_topic,
module: module,
connection_monitor_ref: monitor_ref,
consumer_name: consumer_name
} = gen_state
)
when ref == monitor_ref do
Logger.debug(
"""
#{__MODULE__} for #{stream_name}.#{consumer_name}:
NATS connection has died. PullConsumer is reconnecting.
""",
module: module,
listening_topic: listening_topic,
subscription_id: subscription_id,
connection_name: connection_name
)
# Clear consumer name on reconnect so it gets recreated (for ephemeral and auto-cleanup consumers)
# Clear buffer to avoid processing stale messages from the dead connection
gen_state = %{
gen_state
| consumer_name: nil,
subscription_id: nil,
connection_monitor_ref: nil,
buffer: []
}
{:connect, :reconnect, gen_state}
end
def handle_info(
other,
%__MODULE__{
connection_options: %ConnectionOptions{
connection_name: connection_name,
stream_name: stream_name
},
subscription_id: subscription_id,
listening_topic: listening_topic,
module: module,
consumer_name: consumer_name
} = gen_state
) do
Logger.debug(
"""
#{__MODULE__} for #{stream_name}.#{consumer_name} received
unexpected message: #{inspect(other, pretty: true)}
""",
module: module,
listening_topic: listening_topic,
subscription_id: subscription_id,
connection_name: connection_name
)
{:noreply, gen_state}
end
def handle_call(
:close,
from,
%__MODULE__{
connection_options: %ConnectionOptions{
connection_name: connection_name,
stream_name: stream_name
},
subscription_id: subscription_id,
listening_topic: listening_topic,
module: module,
consumer_name: consumer_name
} = gen_state
) do
Logger.debug("#{__MODULE__} for #{stream_name}.#{consumer_name} received :close call.",
module: module,
listening_topic: listening_topic,
subscription_id: subscription_id,
connection_name: connection_name
)
{:disconnect, {:close, from}, gen_state}
end
defp next_message(conn, stream_name, consumer_name, domain, listening_topic) do
Gnat.Jetstream.API.Consumer.request_next_message(
conn,
stream_name,
consumer_name,
listening_topic,
domain
)
end
defp initial_fetch(gen_state, conn, stream_name, consumer_name, domain, listening_topic) do
if gen_state.connection_options.batch_size > 1 do
request_batch(conn, gen_state, :catching_up)
else
next_message(conn, stream_name, consumer_name, domain, listening_topic)
end
end
defp request_batch(conn, gen_state, mode) do
%{
connection_options: %ConnectionOptions{
stream_name: stream_name,
batch_size: batch_size,
domain: domain,
request_expires: expires
},
consumer_name: consumer_name,
listening_topic: listening_topic
} = gen_state
opts =
case mode do
:catching_up -> [batch: batch_size, no_wait: true]
:tailing -> [batch: batch_size, expires: expires]
end
Gnat.Jetstream.API.Consumer.request_next_message(
conn,
stream_name,
consumer_name,
listening_topic,
domain,
opts
)
end
defp process_and_ack_batch(%{buffer: buffer, module: module, state: state} = gen_state) do
messages = Enum.reverse(buffer)
new_state =
Enum.reduce(messages, state, fn message, acc_state ->
case module.handle_message(message, acc_state) do
{:ack, updated_state} ->
updated_state
{action, updated_state} ->
Logger.warning(
"PullConsumer batch mode does not support #{inspect(action)}, treating as :ack"
)
updated_state
end
end)
# With ack_policy: :all, acking the last message covers the entire batch
last = List.last(messages)
Gnat.Jetstream.ack(last)
%{gen_state | state: new_state, buffer: []}
end
end
================================================
FILE: lib/gnat/jetstream/pull_consumer.ex
================================================
defmodule Gnat.Jetstream.PullConsumer do
@moduledoc """
A behaviour which provides the NATS JetStream Pull Consumer functionalities.
When a Consumer is pull-based, it means that the messages will be delivered when the server
is asked for them.
## Example
Declare a module which uses `Gnat.Jetstream.PullConsumer` and implements `c:init/1` and
`c:handle_message/2` callbacks.
defmodule MyApp.PullConsumer do
use Gnat.Jetstream.PullConsumer
def start_link(arg) do
Jetstream.PullConsumer.start_link(__MODULE__, arg)
end
@impl true
def init(_arg) do
{:ok, nil,
connection_name: :gnat,
stream_name: "TEST_STREAM",
consumer_name: "TEST_CONSUMER"}
end
@impl true
def handle_message(message, state) do
# Do some processing with the message.
{:ack, state}
end
end
You can then place your Pull Consumer in a supervision tree. Remember that you need to have the
`Gnat.ConnectionSupervisor` set up.
defmodule MyApp.Application do
use Application
@impl true
def start(_type, _args) do
children = [
# Create NATS connection
{Gnat.ConnectionSupervisor, ...},
# Start NATS Jetstream Pull Consumer
MyApp.PullConsumer,
]
opts = [strategy: :one_for_one]
Supervisor.start_link(children, opts)
end
end
## Connection Options
In order to establish consumer connection with NATS, you need to pass several connection options
via keyword list in third element of a tuple returned from `c:init/1` callback.
Following options **must** be provided. Omitting this options will cause the process to raise
errors upon initialization:
* `:connection_name` - Gnat connection or `Gnat.ConnectionSupervisor` name/PID.
For **durable consumers**, provide:
* `:stream_name` - name of an existing stream the consumer will consume messages from.
* `:consumer_name` - name of an existing consumer pointing at the stream.
For **ephemeral consumers**, provide:
* `:consumer` - a `Gnat.Jetstream.API.Consumer` struct for creating an ephemeral consumer.
The consumer struct must have `durable_name: nil` OR `inactive_threshold` set to ensure
that the server will clean it up. The `stream_name` field must also be set.
You can also pass the optional ones:
* `:connection_retry_timeout` - a duration in milliseconds after which the PullConsumer which
failed to establish NATS connection retries, defaults to `1000`
* `:connection_retries` - a number of attempts the PullConsumer will make to establish the NATS
connection. When this value is exceeded, the pull consumer stops with the `:timeout` reason,
defaults to `10`
* `:inbox_prefix` - allows the default `_INBOX.` prefix to be customized. Should end with a dot.
* `:domain` - use a JetStream domain, this is mostly used on leaf nodes.
* `:batch_size` - when set to a value greater than 1, enables batch mode. Messages are
buffered and delivered to `c:handle_message/2` in batches. Only the last message per
batch is acknowledged, so the underlying consumer should use `ack_policy: :all` for
correctness. This dramatically improves throughput for consumers that need to catch up
on large backlogs. In batch mode, `:nack` and `:term` returns from `c:handle_message/2`
are treated as `:ack` since `ack_policy: :all` cannot selectively reject messages.
Defaults to `1` (single-message mode).
* `:request_expires` - duration in nanoseconds a batch-mode pull request will linger on
the server while tailing (no backlog) before the server replies with a `408` terminator
and the consumer issues a fresh pull. Only used when `:batch_size` is greater than 1.
Defaults to `5_000_000_000` (5 seconds).
## Dynamic Connection Options
It is possible that you have to determine some of the options dynamically depending on pull
consumer's init argument. To do so, it is recommended to derive these options values from some
init argument:
defmodule MyApp.PullConsumer do
use Gnat.Jetstream.PullConsumer
def start_link() do
Gnat.Jetstream.PullConsumer.start_link(__MODULE__, %{counter: counter})
end
@impl true
def init(%{counter: counter}) do
{:ok, nil,
connection_name: :gnat,
stream_name: "TEST_STREAM_#\{counter}",
consumer_name: "TEST_CONSUMER_#\{counter}"}
end
...
end
## Ephemeral Consumer Example
You can create ephemeral consumers by providing a `:consumer` struct with `durable_name: nil`.
These are automatically created and cleaned up with the connection lifecycle:
defmodule MyApp.EphemeralPullConsumer do
use Gnat.Jetstream.PullConsumer
def start_link(arg) do
Jetstream.PullConsumer.start_link(__MODULE__, arg)
end
@impl true
def init(_arg) do
consumer = %Gnat.Jetstream.API.Consumer{
stream_name: "TEST_STREAM",
durable_name: nil, # Must be nil for ephemeral consumers
filter_subject: "orders.*"
}
{:ok, nil,
connection_name: :gnat,
consumer: consumer}
end
@impl true
def handle_message(message, state) do
# Do some processing with the message.
{:ack, state}
end
end
## Auto-Cleanup Durable Consumer Example
For scenarios where you want persistence across reconnections but automatic cleanup
(e.g., Kubernetes pods), you can create durable consumers with `inactive_threshold`:
defmodule MyApp.AutoCleanupPullConsumer do
use Gnat.Jetstream.PullConsumer
def start_link(pod_name) do
Jetstream.PullConsumer.start_link(__MODULE__, pod_name)
end
@impl true
def init(pod_name) do
consumer = %Gnat.Jetstream.API.Consumer{
stream_name: "ORDERS_STREAM",
durable_name: "orders-consumer-#{System.get_env("POD_NAME")}", # Named after pod
inactive_threshold: 300_000_000_000, # 5 minutes in nanoseconds
filter_subject: "orders.*"
}
{:ok, nil,
connection_name: :gnat,
consumer: consumer}
end
@impl true
def handle_message(message, state) do
# Process order message with state persistence
{:ack, state}
end
end
## How to supervise
A `PullConsumer` is most commonly started under a supervision tree. When we invoke
`use Gnat.Jetstream.PullConsumer`, it automatically defines a `child_spec/1` function that allows us
to start the pull consumer directly under a supervisor. To start a pull consumer under
a supervisor with an initial argument of :example, one may do:
children = [
{MyPullConsumer, :example}
]
Supervisor.start_link(children, strategy: :one_for_all)
While one could also simply pass the `MyPullConsumer` as a child to the supervisor, such as:
children = [
MyPullConsumer # Same as {MyPullConsumer, []}
]
Supervisor.start_link(children, strategy: :one_for_all)
A common approach is to use a keyword list, which allows setting init argument and server options,
for example:
def start_link(opts) do
{initial_state, opts} = Keyword.pop(opts, :initial_state, nil)
Gnat.Jetstream.PullConsumer.start_link(__MODULE__, initial_state, opts)
end
and then you can use `MyPullConsumer`, `{MyPullConsumer, name: :my_consumer}` or even
`{MyPullConsumer, initial_state: :example, name: :my_consumer}` as a child specification.
`use Gnat.Jetstream.PullConsumer` also accepts a list of options which configures the child
specification and therefore how it runs under a supervisor. The generated `child_spec/1` can be
customized with the following options:
* `:id` - the child specification identifier, defaults to the current module
* `:restart` - when the child should be restarted, defaults to `:permanent`
* `:shutdown` - how to shut down the child, either immediately or by giving it time to shut down
For example:
use Gnat.Jetstream.PullConsumer, restart: :transient, shutdown: 10_000
See the "Child specification" section in the `Supervisor` module for more detailed information.
The `@doc` annotation immediately preceding `use Jetstream.PullConsumer` will be attached to
the generated `child_spec/1` function.
## Name registration
A pull consumer is bound to the same name registration rules as GenServers.
Read more about it in the `GenServer` documentation.
"""
@doc """
Invoked when the server is started. `start_link/3` or `start/3` will block until it returns.
`init_arg` is the argument term (second argument) passed to `start_link/3`.
See `c:Connection.init/1` for more details.
"""
@callback init(init_arg :: term) ::
{:ok, state :: term(), connection_options()}
| :ignore
| {:stop, reason :: any}
@doc """
Invoked to synchronously process a message pulled by the consumer.
Depending on the value it returns, the acknowledgement is or is not sent.
Only real stream messages reach this callback. JetStream informational
status messages (e.g. `100` heartbeat, `404`/`408` pull terminator, `409`
leadership change) are intercepted by the consumer and never passed here.
See `c:handle_status/2` if you want to observe them.
## ACK actions
Possible ACK actions values explained:
* `:ack` - acknowledges the message was handled and requests delivery of the next message to
the reply subject.
* `:nack` - signals that the message will not be processed now and processing can move onto
the next message, NAK'd message will be retried.
* `:term` - instructs the server to stop redelivery of a message without acknowledging it as
successfully processed.
* `:noreply` - nothing is sent. You may send later asynchronously an ACK or NACK message using
the `Jetstream.ack/1` or `Jetstream.nack/1` and similar functions from `Jetstream` module.
## Example
def handle_message(message, state) do
IO.inspect(message)
{:ack, state}
end
"""
@callback handle_message(message :: Gnat.message(), state :: term()) ::
{ack_action, new_state}
when ack_action: :ack | :nack | :term | :noreply, new_state: term()
@doc """
Invoked after the consumer has been created or verified on the NATS server.
This callback is called during connection (and reconnection) after the JetStream
consumer has been successfully created or confirmed to exist. It receives the full
consumer info map returned by the server, which includes fields like `num_pending`
(the number of messages waiting to be delivered).
This is useful for detecting the initial state of the consumer. For example, if
`num_pending` is `0`, you know there are no existing messages to replay and can
mark the consumer as caught up immediately.
Returning `{:ok, state}` allows you to update the consumer's state based on the
consumer info.
This callback is optional. If not implemented, the state is passed through unchanged.
## Example
@impl true
def handle_connected(consumer_info, state) do
if consumer_info.num_pending == 0 do
{:ok, mark_as_loaded(state)}
else
{:ok, state}
end
end
"""
@callback handle_connected(
consumer_info :: Gnat.Jetstream.API.Consumer.info(),
state :: term()
) :: {:ok, new_state :: term()}
@doc """
Invoked when the consumer receives an informational JetStream status message
instead of a real stream message.
JetStream delivers status messages on the same subscription as regular
messages — for example a `100` idle heartbeat, a `404`/`408` pull request
terminator, or a `409` leadership change. These are not stream records and
cannot be acked, so the PullConsumer never forwards them to `c:handle_message/2`.
By default they are silently dropped and the consumer continues fetching
the next message. Implement this callback if you want to observe them — for
example to log a warning on leadership changes, or to track heartbeat
arrival.
The callback receives the raw `Gnat.message()` (which includes `:status`
and optionally `:description`) and the current state. Returning
`{:ok, new_state}` updates the state; the consumer then proceeds the same
way it would have if the callback had not been defined.
This callback is optional.
## Example
@impl true
def handle_status(%{status: "409", description: description}, state) do
Logger.warning("JetStream 409 from consumer: #\{description}")
{:ok, state}
end
def handle_status(_message, state), do: {:ok, state}
"""
@callback handle_status(message :: Gnat.message(), state :: term()) ::
{:ok, new_state :: term()}
@optional_callbacks [handle_connected: 2, handle_status: 2]
@typedoc """
The pull consumer reference.
"""
@type consumer :: GenServer.server()
@typedoc """
Connection option values used to connect the consumer to NATS server.
"""
@type connection_option ::
{:connection_name, GenServer.server()}
| {:stream_name, String.t()}
| {:consumer_name, String.t()}
| {:consumer, Gnat.Jetstream.API.Consumer.t()}
| {:connection_retry_timeout, non_neg_integer()}
| {:connection_retries, non_neg_integer()}
| {:domain, String.t()}
| {:batch_size, pos_integer()}
| {:request_expires, non_neg_integer()}
@typedoc """
Connection options used to connect the consumer to NATS server.
"""
@type connection_options :: [connection_option()]
defmacro __using__(opts) do
quote location: :keep, bind_quoted: [opts: opts] do
@behaviour Gnat.Jetstream.PullConsumer
unless Module.has_attribute?(__MODULE__, :doc) do
@doc """
Returns a specification to start this module under a supervisor.
See the "Child specification" section in the `Supervisor` module for more detailed
information.
"""
end
@spec child_spec(arg :: GenServer.options()) :: Supervisor.child_spec()
def child_spec(arg) do
default = %{
id: __MODULE__,
start: {__MODULE__, :start_link, [arg]}
}
Supervisor.child_spec(default, unquote(Macro.escape(opts)))
end
defoverridable child_spec: 1
end
end
@doc """
Starts a pull consumer linked to the current process with the given function.
This is often used to start the pull consumer as part of a supervision tree.
Once the server is started, the `c:init/1` function of the given `module` is called with
`init_arg` as its argument to initialize the server. To ensure a synchronized start-up procedure,
this function does not return until `c:init/1` has returned.
See `GenServer.start_link/3` for more details.
"""
@spec start_link(module(), init_arg :: term(), option
gitextract_y72cyuck/
├── .dialyzer_ignore.exs
├── .formatter.exs
├── .github/
│ └── workflows/
│ └── CI.yml
├── .gitignore
├── CHANGELOG.md
├── LICENSE.txt
├── MAINTAINERS.md
├── README.md
├── bench/
│ ├── client.exs
│ ├── kv_consume.exs
│ ├── parse.exs
│ ├── publish.exs
│ ├── request.exs
│ ├── request_multi.exs
│ ├── server.exs
│ └── service_bench.exs
├── dependencies.md
├── docs/
│ └── js/
│ ├── guides/
│ │ ├── broadway.md
│ │ ├── managing.md
│ │ └── push_based_consumer.md
│ └── introduction/
│ ├── getting_started.md
│ └── overview.md
├── lib/
│ ├── gnat/
│ │ ├── command.ex
│ │ ├── connection_supervisor.ex
│ │ ├── consumer_supervisor.ex
│ │ ├── handshake.ex
│ │ ├── jetstream/
│ │ │ ├── api/
│ │ │ │ ├── consumer.ex
│ │ │ │ ├── kv/
│ │ │ │ │ ├── entry.ex
│ │ │ │ │ └── watcher.ex
│ │ │ │ ├── kv.ex
│ │ │ │ ├── message.ex
│ │ │ │ ├── object/
│ │ │ │ │ └── meta.ex
│ │ │ │ ├── object.ex
│ │ │ │ ├── stream.ex
│ │ │ │ └── util.ex
│ │ │ ├── jetstream.ex
│ │ │ ├── pager.ex
│ │ │ ├── pull_consumer/
│ │ │ │ ├── connection_options.ex
│ │ │ │ └── server.ex
│ │ │ └── pull_consumer.ex
│ │ ├── parsec.ex
│ │ ├── server.ex
│ │ └── services/
│ │ ├── server.ex
│ │ ├── service.ex
│ │ ├── service_responder.ex
│ │ └── wire_protocol.ex
│ └── gnat.ex
├── mix.exs
├── scripts/
│ └── cluster/
│ ├── cluster.sh
│ ├── driver.exs
│ ├── n1.conf
│ ├── n2.conf
│ └── n3.conf
└── test/
├── command_test.exs
├── fixtures/
│ ├── ca.pem
│ ├── client-cert.pem
│ ├── client-key.pem
│ ├── nkey_config
│ ├── nkey_seed
│ ├── server-cert.pem
│ └── server-key.pem
├── gnat/
│ ├── consumer_supervisor_test.exs
│ ├── handshake_test.exs
│ ├── parsec_property_test.exs
│ └── parsec_test.exs
├── gnat_property_test.exs
├── gnat_test.exs
├── jetstream/
│ ├── api/
│ │ ├── consumer_doc_test.exs
│ │ ├── consumer_test.exs
│ │ ├── kv/
│ │ │ ├── entry_test.exs
│ │ │ └── watcher_test.exs
│ │ ├── kv_test.exs
│ │ ├── object_test.exs
│ │ ├── stream_doc_test.exs
│ │ └── stream_test.exs
│ ├── message_test.exs
│ └── pager_test.exs
├── pull_consumer/
│ ├── batch_test.exs
│ ├── connectivity_test.exs
│ ├── ephemeral_test.exs
│ ├── status_messages_test.exs
│ └── using_macro_test.exs
├── support/
│ ├── conn_case.ex
│ └── generators.ex
└── test_helper.exs
SYMBOL INDEX (493 symbols across 58 files)
FILE: bench/client.exs
class Client (line 5) | defmodule Client
method setup (line 8) | def setup(id) do
method send_request (line 14) | def send_request(gnat, request) do
method send_requests (line 18) | def send_requests(gnat, how_many, request) do
class Benchmark (line 27) | defmodule Benchmark
method benchmark (line 28) | def benchmark(num_actors, requests_per_actor, request) do
method record_rpc_time (line 37) | def record_rpc_time(micro_seconds) do
method print_statistics (line 41) | def print_statistics(total_requests, total_bytes, total_micros) do
method time_benchmark (line 66) | def time_benchmark(num_actors, requests_per_actor, request) do
method wait_for_times (line 82) | def wait_for_times(0), do: :done
method wait_for_times (line 83) | def wait_for_times(n) do
FILE: bench/kv_consume.exs
class BenchBatchPullConsumer (line 28) | defmodule BenchBatchPullConsumer
method start (line 31) | def start(args) do
method init (line 36) | def init(%{tab: tab, notify: pid, expected: expected, batch_size: batc...
method handle_message (line 50) | def handle_message(message, state) do
class KVConsumeBench (line 62) | defmodule KVConsumeBench
method setup (line 67) | def setup(conn, count) do
method pager_consume (line 96) | def pager_consume(conn, batch_size) do
method pull_ack_next_consume (line 113) | def pull_ack_next_consume(conn, batch_size) do
method receive_with_ack_next (line 152) | defp receive_with_ack_next(_sub, _inbox, _tab, total, total), do: :ok
method receive_with_ack_next (line 154) | defp receive_with_ack_next(sub, inbox, tab, count, total) do
method batch_pull_consumer_consume (line 185) | def batch_pull_consumer_consume(expected, batch_size) do
FILE: bench/request.exs
class EchoServer (line 1) | defmodule EchoServer
method run (line 2) | def run(gnat) do
method init (line 6) | def init(gnat) do
method loop (line 11) | def loop(gnat) do
FILE: bench/request_multi.exs
class EchoServer (line 1) | defmodule EchoServer
method run (line 2) | def run(gnat) do
method init (line 6) | def init(gnat) do
method loop (line 11) | def loop(gnat) do
FILE: bench/server.exs
class EchoServer (line 16) | defmodule EchoServer
method handle (line 17) | def handle(%{body: body, reply_to: reply_to, gnat: gnat_pid}) do
method wait_loop (line 21) | def wait_loop do
FILE: bench/service_bench.exs
class EchoService (line 1) | defmodule EchoService
method request (line 4) | def request(%{body: body}, "echo", _group) do
method definition (line 8) | def definition do
FILE: lib/gnat.ex
class Gnat (line 5) | defmodule Gnat
method start_link (line 153) | def start_link(connection_settings \\ %{}, opts \\ []) do
method stop (line 166) | def stop(pid), do: GenServer.call(pid, :stop)
method sub (line 191) | def sub(pid, subscriber, topic, opts \\ []) do
method pub (line 228) | def pub(pid, topic, message, opts \\ []) do
method request (line 264) | def request(pid, topic, body, opts \\ []) do
method request_multi (line 303) | def request_multi(pid, topic, body, opts \\ []) do
method unsub (line 352) | def unsub(pid, sid, opts \\ []) do
method ping (line 363) | def ping(_pid) do
method active_subscriptions (line 369) | def active_subscriptions(pid) do
method server_info (line 377) | def server_info(name) do
method init (line 382) | def init(connection_settings) do
method handle_info (line 414) | def handle_info(:ping_check, %{waiting_on_pong: true} = state) do
method handle_info (line 422) | def handle_info(:ping_check, %{waiting_on_pong: false} = state) do
method handle_info (line 428) | def handle_info({:tcp, socket, data}, %{socket: socket} = state) do
method handle_info (line 441) | def handle_info({:ssl, socket, data}, state) do
method handle_info (line 445) | def handle_info({:tcp_closed, _}, state) do
method handle_info (line 449) | def handle_info({:ssl_closed, _}, state) do
method handle_info (line 453) | def handle_info({:tcp_error, _, reason}, state) do
method handle_info (line 457) | def handle_info({:DOWN, _ref, :process, pid, _reason}, state) do
method handle_info (line 466) | def handle_info(other, state) do
method handle_call (line 472) | def handle_call(:stop, _from, state) do
method handle_call (line 477) | def handle_call({:sub, receiver, topic, opts}, _from, %{next_sid: sid}...
method handle_call (line 488) | def handle_call({:pub, topic, message, opts}, from, state) do
method handle_call (line 497) | def handle_call({:request, request}, _from, state) do
method handle_call (line 529) | def handle_call({:unsub, sid, opts}, _from, state) do
method handle_call (line 534) | def handle_call({:ping, pinger}, _from, state) do
method handle_call (line 539) | def handle_call(:active_subscriptions, _from, state) do
method handle_call (line 544) | def handle_call(:server_info, _from, state) do
method unsub_sid (line 548) | defp unsub_sid(sid, opts, state) do
method create_request_subscription (line 562) | defp create_request_subscription(%{request_inbox_prefix: request_inbox...
method make_new_inbox (line 570) | defp make_new_inbox(%{request_inbox_prefix: prefix}), do: prefix <> nu...
method nuid (line 572) | defp nuid(), do: :crypto.strong_rand_bytes(12) |> Base.encode64()
method prepare_headers (line 574) | defp prepare_headers(opts) do
method socket_close (line 583) | defp socket_close(%{socket: socket, connection_settings: %{tls: true}}...
method socket_close (line 584) | defp socket_close(%{socket: socket}), do: :gen_tcp.close(socket)
method socket_write (line 586) | defp socket_write(%{socket: socket, connection_settings: %{tls: true}}...
method socket_write (line 590) | defp socket_write(%{socket: socket}, iodata), do: :gen_tcp.send(socket...
method add_subscription_to_state (line 592) | defp add_subscription_to_state(%{receivers: receivers} = state, sid, p...
method cleanup_subscription_from_state (line 599) | defp cleanup_subscription_from_state(%{receivers: receivers} = state, ...
method cleanup_subscription_from_state (line 604) | defp cleanup_subscription_from_state(%{receivers: receivers} = state, ...
method process_message (line 609) | defp process_message({:info, server_info}, state) do
method process_message (line 613) | defp process_message({:msg, topic, @request_sid, reply_to, body}, stat...
method process_message (line 627) | defp process_message({:msg, topic, sid, reply_to, body}, state) do
method process_message (line 643) | defp process_message(
method process_message (line 666) | defp process_message({:hmsg, topic, sid, reply_to, status, description...
method process_message (line 689) | defp process_message(:ping, state) do
method process_message (line 694) | defp process_message(:pong, state) do
method process_message (line 698) | defp process_message({:error, message}, state) do
method receive_additional_pubs (line 707) | defp receive_additional_pubs(commands, froms, 0), do: {commands, froms}
method receive_additional_pubs (line 709) | defp receive_additional_pubs(commands, froms, how_many_more) do
method receive_additional_tcp_data (line 720) | defp receive_additional_tcp_data(_socket, packets, 0), do: Enum.revers...
method receive_additional_tcp_data (line 722) | defp receive_additional_tcp_data(socket, packets, n) do
method update_subscriptions_after_delivering_message (line 731) | defp update_subscriptions_after_delivering_message(%{receivers: receiv...
method receive_multi_request_responses (line 742) | defp receive_multi_request_responses(_sub, _exp, 0), do: []
method receive_multi_request_responses (line 744) | defp receive_multi_request_responses(subscription, expiration, max_mes...
method receive_request_response (line 765) | defp receive_request_response(subscription, timeout) do
method schedule_ping_check (line 778) | defp schedule_ping_check(connection_settings) do
FILE: lib/gnat/command.ex
class Gnat.Command (line 1) | defmodule Gnat.Command
method build (line 10) | def build(:pub, topic, payload, []),
method build (line 13) | def build(:pub, topic, payload, reply_to: reply),
method build (line 26) | def build(:pub, topic, payload, headers: headers) do
method build (line 48) | def build(:pub, topic, payload, headers: headers, reply_to: reply) do
method build (line 72) | def build(:sub, topic, sid, []), do: [@sub, " ", topic, " ", Integer.t...
method build (line 74) | def build(:sub, topic, sid, queue_group: qg),
method build (line 77) | def build(:unsub, sid, []), do: [@unsub, " #{sid}", @newline]
method build (line 78) | def build(:unsub, sid, max_messages: max), do: [@unsub, " #{sid}", " #...
FILE: lib/gnat/connection_supervisor.ex
class Gnat.ConnectionSupervisor (line 1) | defmodule Gnat.ConnectionSupervisor
method start_link (line 40) | def start_link(settings, options \\ []) do
method init (line 45) | def init(options) do
method handle_info (line 59) | def handle_info(:attempt_connection, state) do
method handle_info (line 77) | def handle_info({:EXIT, _pid, _reason}, %{gnat: nil} = state) do
method handle_info (line 81) | def handle_info({:EXIT, _pid, reason}, state) do
method handle_info (line 87) | def handle_info(msg, state) do
method random_connection_config (line 92) | defp random_connection_config(%{connection_settings: connection_settin...
FILE: lib/gnat/consumer_supervisor.ex
class Gnat.ConsumerSupervisor (line 1) | defmodule Gnat.ConsumerSupervisor
method start_link (line 63) | def start_link(settings, options \\ []) do
method init (line 68) | def init(settings) do
method handle_info (line 94) | def handle_info(:connect, %{connection_name: name} = state) do
method handle_info (line 114) | def handle_info(
method handle_info (line 123) | def handle_info({:DOWN, _ref, :process, _task_pid, _reason}, state), d...
method handle_info (line 126) | def handle_info(
method handle_info (line 134) | def handle_info({:msg, gnat_message}, %{service: service, module: modu...
method handle_info (line 144) | def handle_info({:msg, gnat_message}, %{module: module} = state) do
method handle_info (line 153) | def handle_info({:msg, gnat_message}, %{consuming_function: {mod, fun}...
method handle_info (line 158) | def handle_info(other, state) do
method terminate (line 164) | def terminate(:shutdown, state) do
method terminate (line 178) | def terminate(reason, _state) do
method receive_final_broker_messages (line 182) | defp receive_final_broker_messages(state) do
method wait_for_empty_task_supervisor (line 193) | defp wait_for_empty_task_supervisor(%{task_supervisor_pid: pid} = stat...
method subscribe_to_topics (line 205) | defp subscribe_to_topics(%{service: service}, connection_pid) do
method subscribe_to_topics (line 218) | defp subscribe_to_topics(state, connection_pid) do
method maybe_append_service (line 232) | defp maybe_append_service(state, %{service_definition: config}) do
method maybe_append_service (line 242) | defp maybe_append_service(state, _), do: {:ok, state}
method maybe_append_module (line 244) | defp maybe_append_module(state, %{module: module}) do
method maybe_append_module (line 248) | defp maybe_append_module(state, _), do: {:ok, state}
method maybe_append_consuming_function (line 250) | defp maybe_append_consuming_function(state, %{consuming_function: cons...
method maybe_append_consuming_function (line 254) | defp maybe_append_consuming_function(state, _), do: {:ok, state}
method validate_state (line 256) | defp validate_state(state) do
FILE: lib/gnat/handshake.ex
class Gnat.Handshake (line 1) | defmodule Gnat.Handshake
method connect (line 9) | def connect(settings) do
method negotiate_settings (line 18) | def negotiate_settings(server_settings, user_settings) do
method perform_handshake (line 27) | defp perform_handshake(tcp, user_settings) do
method send_connect (line 41) | defp send_connect(%{tls: true}, settings, socket) do
method send_connect (line 45) | defp send_connect(_, settings, socket) do
method negotiate_auth (line 49) | defp negotiate_auth(
method negotiate_auth (line 58) | defp negotiate_auth(settings, _server, %{token: token} = _user, true =...
method negotiate_auth (line 62) | defp negotiate_auth(
method negotiate_auth (line 74) | defp negotiate_auth(
method negotiate_auth (line 87) | defp negotiate_auth(settings, _server, _user, _auth_required) do
method negotiate_headers (line 91) | defp negotiate_headers(settings, %{headers: true} = _server, user_sett...
method negotiate_headers (line 99) | defp negotiate_headers(_settings, _server, %{headers: true} = _user) do
method negotiate_headers (line 103) | defp negotiate_headers(settings, _server, _user) do
method negotiate_no_responders (line 107) | defp negotiate_no_responders(%{headers: true} = settings, _server_sett...
method negotiate_no_responders (line 113) | defp negotiate_no_responders(settings, _server_settings, _user_setting...
method upgrade_connection (line 117) | defp upgrade_connection(tcp, %{tls: true, ssl_opts: opts}) do
method upgrade_connection (line 122) | defp upgrade_connection(tcp, _settings), do: {:ok, tcp}
FILE: lib/gnat/jetstream/api/consumer.ex
class Gnat.Jetstream.API.Consumer (line 1) | defmodule Gnat.Jetstream.API.Consumer
method create (line 241) | def create(conn, %__MODULE__{} = consumer) do
method delete (line 270) | def delete(conn, stream_name, consumer_name, domain \\ nil) do
method info (line 297) | def info(conn, stream_name, consumer_name, domain \\ nil) do
method list (line 322) | def list(conn, stream_name, params \\ []) do
method request_next_message (line 377) | def request_next_message(
method js_api (line 411) | defp js_api(nil), do: "$JS.API"
method js_api (line 412) | defp js_api(""), do: "$JS.API"
method js_api (line 413) | defp js_api(domain), do: "$JS.#{domain}.API"
method create_payload (line 415) | defp create_payload(%__MODULE__{} = cons) do
method to_config (line 447) | defp to_config(raw) do
method to_info (line 475) | defp to_info(raw) do
method validate (line 498) | defp validate(consumer) do
method validate_durable (line 527) | defp validate_durable(consumer) do
FILE: lib/gnat/jetstream/api/kv.ex
class Gnat.Jetstream.API.KV (line 1) | defmodule Gnat.Jetstream.API.KV
method create_bucket (line 52) | def create_bucket(conn, bucket_name, params \\ []) do
method adjust_duplicate_window (line 83) | defp adjust_duplicate_window(_ttl), do: @two_minutes_in_nanoseconds
method delete_bucket (line 93) | def delete_bucket(conn, bucket_name) do
method create_key (line 116) | def create_key(conn, bucket_name, key, value, opts \\ []) do
method delete_key (line 141) | def delete_key(conn, bucket_name, key, opts \\ []) do
method purge_key (line 170) | def purge_key(conn, bucket_name, key, opts \\ []) do
method put_value (line 200) | def put_value(conn, bucket_name, key, value, opts \\ []) do
method get_value (line 220) | def get_value(conn, bucket_name, key) do
method contents (line 244) | def contents(conn, bucket_name, opts \\ []) do
method keys (line 282) | def keys(conn, bucket_name, opts \\ []) do
method info (line 324) | def info(conn, bucket_name, opts \\ []) do
method watch (line 338) | def watch(conn, bucket_name, handler) do
method unwatch (line 354) | def unwatch(pid) do
method is_kv_bucket_stream? (line 360) | def is_kv_bucket_stream?(stream_name) do
method kv_bucket_stream? (line 371) | def kv_bucket_stream?(stream_name) do
method list_buckets (line 379) | def list_buckets(conn) do
method stream_name (line 394) | def stream_name(bucket_name) do
method stream_subjects (line 398) | defp stream_subjects(bucket_name) do
method key_name (line 402) | defp key_name(bucket_name, key) do
method subject_to_key (line 407) | def subject_to_key(subject, bucket_name) do
FILE: lib/gnat/jetstream/api/kv/entry.ex
class Gnat.Jetstream.API.KV.Entry (line 1) | defmodule Gnat.Jetstream.API.KV.Entry
method from_message (line 95) | def from_message(message, bucket_name) do
method status_message? (line 112) | defp status_message?(_), do: false
method extract_key (line 114) | defp extract_key(%{topic: topic}, bucket_name) do
method operation (line 133) | defp operation(_message), do: :put
method apply_metadata (line 135) | defp apply_metadata(entry, message) do
FILE: lib/gnat/jetstream/api/kv/watcher.ex
class Gnat.Jetstream.API.KV.Watcher (line 1) | defmodule Gnat.Jetstream.API.KV.Watcher
method start_link (line 29) | def start_link(opts) do
method stop (line 33) | def stop(pid) do
method init (line 37) | def init(opts) do
method terminate (line 51) | def terminate(_reason, state) do
method handle_info (line 74) | def handle_info({:msg, message}, state) do
method action (line 86) | defp action(:put), do: :key_added
method action (line 87) | defp action(:delete), do: :key_deleted
method action (line 88) | defp action(:purge), do: :key_purged
method subscribe (line 90) | defp subscribe(conn, bucket_name) do
FILE: lib/gnat/jetstream/api/message.ex
class Gnat.Jetstream.API.Message (line 1) | defmodule Gnat.Jetstream.API.Message
method metadata (line 35) | def metadata(%{reply_to: "$JS.ACK." <> ack_topic}),
method metadata (line 38) | def metadata(_), do: {:error, :no_jetstream_message}
method decode_reply_to (line 42) | defp decode_reply_to([
method decode_reply_to (line 76) | defp decode_reply_to([
method decode_reply_to (line 106) | defp decode_reply_to(_), do: {:error, :invalid_ack_reply_to}
class Metadata (line 10) | defmodule Metadata
FILE: lib/gnat/jetstream/api/object.ex
class Gnat.Jetstream.API.Object (line 1) | defmodule Gnat.Jetstream.API.Object
method create_bucket (line 23) | def create_bucket(conn, bucket_name, params \\ []) do
method delete_bucket (line 45) | def delete_bucket(conn, bucket_name) do
method delete (line 50) | def delete(conn, bucket_name, object_name) do
method get (line 62) | def get(conn, bucket_name, object_name, chunk_fun) do
method info (line 70) | def info(conn, bucket_name, object_name) do
method list (line 88) | def list(conn, bucket_name, options \\ []) do
method put (line 119) | def put(conn, bucket_name, object_name, io) do
method is_object_bucket_stream? (line 154) | def is_object_bucket_stream?(stream_name) do
method list_buckets (line 162) | def list_buckets(conn) do
method stream_name (line 181) | defp stream_name(bucket_name) do
method stream_subjects (line 185) | defp stream_subjects(bucket_name) do
method chunk_stream_subject (line 192) | defp chunk_stream_subject(bucket_name) do
method chunk_stream_topic (line 196) | defp chunk_stream_topic(bucket_name, nuid) do
method chunk_stream_topic (line 200) | defp chunk_stream_topic(%Meta{bucket: bucket, nuid: nuid}) do
method meta_stream_subject (line 204) | defp meta_stream_subject(bucket_name) do
method meta_stream_topic (line 208) | defp meta_stream_topic(bucket_name, object_name) do
method adjust_duplicate_window (line 217) | defp adjust_duplicate_window(_ttl), do: @two_minutes_in_nanoseconds
method json_to_meta (line 219) | defp json_to_meta(json) do
method purge_prior_chunks (line 242) | defp purge_prior_chunks(conn, bucket, name) do
method receive_all_metas (line 255) | defp receive_all_metas(sid, num_pending, messages \\ [])
method receive_all_metas (line 257) | defp receive_all_metas(_sid, 0, messages) do
method receive_all_metas (line 261) | defp receive_all_metas(sid, remaining, messages) do
method receive_chunks (line 272) | defp receive_chunks(conn, %Meta{} = meta, chunk_fun) do
method receive_chunks (line 298) | defp receive_chunks(_conn, _sub, 0, _chunk_fun) do
method receive_chunks (line 302) | defp receive_chunks(conn, sub, remaining, chunk_fun) do
method send_chunks (line 325) | defp send_chunks(conn, io, topic) do
method send_chunks (line 332) | defp send_chunks(conn, io, topic, sha, size, chunks) do
method validate_bucket_name (line 356) | defp validate_bucket_name(name) do
FILE: lib/gnat/jetstream/api/object/meta.ex
class Gnat.Jetstream.API.Object.Meta (line 1) | defmodule Gnat.Jetstream.API.Object.Meta
FILE: lib/gnat/jetstream/api/stream.ex
class Gnat.Jetstream.API.Stream (line 1) | defmodule Gnat.Jetstream.API.Stream
method create (line 287) | def create(conn, %__MODULE__{} = stream) do
method update (line 309) | def update(conn, %__MODULE__{} = stream) do
method list (line 419) | def list(conn, params \\ []) do
method js_api (line 475) | defp js_api(nil), do: "$JS.API"
method js_api (line 476) | defp js_api(""), do: "$JS.API"
method js_api (line 477) | defp js_api(domain), do: "$JS.#{domain}.API"
method to_state (line 479) | defp to_state(state) do
method to_stream (line 496) | defp to_stream(stream) do
method to_info (line 531) | defp to_info(%{"config" => config, "state" => state, "created" => crea...
method validate (line 544) | defp validate(stream_settings) do
method validate_message_access_method (line 569) | defp validate_message_access_method(method) do
method validate_purge_method (line 581) | defp validate_purge_method(_) do
FILE: lib/gnat/jetstream/api/util.ex
class Gnat.Jetstream.API.Util (line 1) | defmodule Gnat.Jetstream.API.Util
method request (line 6) | def request(conn, topic, payload) do
method to_datetime (line 19) | def to_datetime(nil), do: nil
method to_datetime (line 21) | def to_datetime(str) do
method to_sym (line 26) | def to_sym(nil), do: nil
method put_if_exist (line 32) | def put_if_exist(target_map, target_key, source_map, source_key) do
method valid_name? (line 39) | def valid_name?(name) do
method invalid_name_message (line 43) | def invalid_name_message do
method decode_base64 (line 47) | def decode_base64(nil), do: nil
method decode_base64 (line 48) | def decode_base64(data), do: Base.decode64!(data)
method reply_inbox (line 50) | def reply_inbox(prefix \\ @default_inbox_prefix)
method reply_inbox (line 51) | def reply_inbox(nil), do: reply_inbox()
method reply_inbox (line 52) | def reply_inbox(prefix), do: prefix <> nuid()
method nuid (line 54) | def nuid() do
FILE: lib/gnat/jetstream/jetstream.ex
class Gnat.Jetstream (line 1) | defmodule Gnat.Jetstream
method ack (line 13) | def ack(%{reply_to: nil}) do
method ack (line 17) | def ack(%{gnat: gnat, reply_to: reply_to}) do
method ack_next (line 28) | def ack_next(%{reply_to: nil}, _consumer_subject) do
method ack_next (line 32) | def ack_next(%{gnat: gnat, reply_to: reply_to}, consumer_subject) do
method nack (line 43) | def nack(%{reply_to: nil}) do
method nack (line 47) | def nack(%{gnat: gnat, reply_to: reply_to}) do
method ack_progress (line 58) | def ack_progress(%{reply_to: nil}) do
method ack_progress (line 62) | def ack_progress(%{gnat: gnat, reply_to: reply_to}) do
method ack_term (line 73) | def ack_term(%{reply_to: nil}) do
method ack_term (line 77) | def ack_term(%{gnat: gnat, reply_to: reply_to}) do
FILE: lib/gnat/jetstream/pager.ex
class Gnat.Jetstream.Pager (line 1) | defmodule Gnat.Jetstream.Pager
method init (line 35) | def init(conn, stream_name, opts) do
method page (line 71) | def page(state) do
method cleanup (line 77) | def cleanup(%{conn: conn} = state) do
method reduce (line 102) | def reduce(conn, stream_name, opts, initial_state, fun) do
method page_through (line 108) | defp page_through(pager, state, fun) do
method request_next_message (line 121) | defp request_next_message(state) do
method receive_messages (line 141) | defp receive_messages(%{sub: sid} = state, messages) do
method apply_opts_to_consumer (line 155) | defp apply_opts_to_consumer(consumer = %Consumer{}, opts) do
FILE: lib/gnat/jetstream/pull_consumer.ex
class Gnat.Jetstream.PullConsumer (line 1) | defmodule Gnat.Jetstream.PullConsumer
method close (line 454) | def close(consumer) do
FILE: lib/gnat/jetstream/pull_consumer/connection_options.ex
class Gnat.Jetstream.PullConsumer.ConnectionOptions (line 1) | defmodule Gnat.Jetstream.PullConsumer.ConnectionOptions
method validate! (line 27) | def validate!(connection_options) do
FILE: lib/gnat/jetstream/pull_consumer/server.ex
class Gnat.Jetstream.PullConsumer.Server (line 1) | defmodule Gnat.Jetstream.PullConsumer.Server
method init (line 23) | def init(%{module: module, init_arg: init_arg}) do
method connect (line 50) | def connect(
method disconnect (line 136) | def disconnect(
method ensure_consumer_exists (line 173) | defp ensure_consumer_exists(gnat, stream_name, consumer_name, nil, dom...
method ensure_consumer_exists (line 186) | defp ensure_consumer_exists(gnat, _stream_name, nil, consumer_struct, ...
method validate_consumer_for_creation (line 199) | defp validate_consumer_for_creation(consumer_definition) do
method validate_batch_ack_policy (line 228) | defp validate_batch_ack_policy(_connection_options, _consumer_info), d...
method maybe_handle_connected (line 230) | defp maybe_handle_connected(module, consumer_info, state) do
method maybe_handle_status (line 239) | defp maybe_handle_status(message, %__MODULE__{module: module, state: s...
method connection_pid (line 256) | defp connection_pid(connection_name) do
method handle_info (line 352) | def handle_info(
method handle_info (line 466) | def handle_info(
method handle_call (line 493) | def handle_call(
method next_message (line 517) | defp next_message(conn, stream_name, consumer_name, domain, listening_...
method initial_fetch (line 527) | defp initial_fetch(gen_state, conn, stream_name, consumer_name, domain...
method request_batch (line 535) | defp request_batch(conn, gen_state, mode) do
method process_and_ack_batch (line 563) | defp process_and_ack_batch(%{buffer: buffer, module: module, state: st...
FILE: lib/gnat/parsec.ex
class Gnat.Parsec (line 1) | defmodule Gnat.Parsec
method new (line 100) | def new, do: %__MODULE__{}
method parse (line 102) | def parse(%__MODULE__{partial: nil} = state, string) do
method parse (line 107) | def parse(%__MODULE__{partial: partial} = state, string) do
method parse_commands (line 112) | def parse_commands("", list), do: {nil, Enum.reverse(list)}
method parse_commands (line 114) | def parse_commands(str, list) do
method parse_command (line 122) | def parse_command(string) do
method parse_headers (line 150) | def parse_headers("NATS/1.0" <> rest) do
method parse_headers (line 178) | def parse_headers(_other) do
method finish_msg (line 182) | def finish_msg(subject, sid, reply_to, length, rest, string) do
method finish_hmsg (line 192) | def finish_hmsg(subject, sid, reply_to, header_length, total_length, r...
FILE: lib/gnat/server.ex
class Gnat.Server (line 1) | defmodule Gnat.Server
method execute (line 61) | def execute(module, message) do
method execute_error (line 76) | defp execute_error(module, message, error) do
method send_reply (line 106) | def send_reply(_other, _iodata) do
FILE: lib/gnat/services/server.ex
class Gnat.Services.Server (line 1) | defmodule Gnat.Services.Server
method execute (line 113) | def execute(_module, %{topic: "$SRV" <> _} = message, service) do
method execute (line 117) | def execute(module, message, service) do
method execute_error (line 153) | defp execute_error(module, message, error) do
method send_reply (line 183) | def send_reply(_other, _iodata) do
FILE: lib/gnat/services/service.ex
class Gnat.Services.Service (line 1) | defmodule Gnat.Services.Service
method init (line 17) | def init(configuration) do
method info (line 33) | def info(service) do
method ping (line 49) | def ping(service) do
method record_request (line 58) | def record_request(%{counters: counters} = _endpoint, elapsed_micros) do
method record_error (line 63) | def record_error(%{counters: counters} = _endpoint, elapsed_micros) do
method subscription_topics_with_queue_group (line 68) | def subscription_topics_with_queue_group(service) do
method stats (line 79) | def stats(service) do
method build_subject_map (line 94) | defp build_subject_map(endpoints) do
method derive_subscription_subject (line 112) | defp derive_subscription_subject(endpoint) do
method endpoint_info (line 128) | defp endpoint_info(endpoint) do
method endpoint_stats (line 137) | defp endpoint_stats(%{counters: counters} = endpoint) do
method validate_configuration (line 168) | defp validate_configuration(configuration) do
method valid_version? (line 196) | defp valid_version?(service_definition) do
method valid_name? (line 206) | defp valid_name?(service_definition) do
method valid_metadata? (line 216) | defp valid_metadata?(nil), do: :ok
method valid_metadata? (line 218) | defp valid_metadata?(md) do
method valid_endpoint? (line 232) | defp valid_endpoint?(endpoint_definition) do
FILE: lib/gnat/services/service_responder.ex
class Gnat.Services.ServiceResponder (line 1) | defmodule Gnat.Services.ServiceResponder
method maybe_respond (line 11) | def maybe_respond(%{topic: topic} = message, service) do
method handle_ping (line 27) | defp handle_ping(tail, service, %{reply_to: rt, gnat: gnat}) do
method handle_info (line 34) | defp handle_info(tail, service, %{reply_to: rt, gnat: gnat}) do
method handle_stats (line 41) | defp handle_stats(tail, service, %{reply_to: rt, gnat: gnat}) do
method should_respond? (line 49) | defp should_respond?(tail, service_name, instance_id) do
FILE: lib/gnat/services/wire_protocol.ex
class Gnat.Services.WireProtocol (line 1) | defmodule Gnat.Services.WireProtocol
class InfoResponse (line 4) | defmodule InfoResponse
class PingResponse (line 38) | defmodule PingResponse
class StatsResponse (line 60) | defmodule StatsResponse
FILE: mix.exs
class Gnat.Mixfile (line 1) | defmodule Gnat.Mixfile
method project (line 7) | def project do
method application (line 27) | def application do
method deps (line 31) | defp deps do
method docs (line 46) | defp docs do
method elixirc_paths (line 68) | defp elixirc_paths(:test), do: ["lib", "test/support"]
method elixirc_paths (line 69) | defp elixirc_paths(_), do: ["lib"]
method package (line 71) | defp package do
FILE: scripts/cluster/driver.exs
class FailoverConsumer (line 88) | defmodule FailoverConsumer
method start_link (line 92) | def start_link(opts), do: Gnat.Jetstream.PullConsumer.start_link(__MOD...
method init (line 95) | def init(opts) do
method handle_message (line 100) | def handle_message(%{body: body, reply_to: reply_to}, state) do
method handle_status (line 113) | def handle_status(%{status: status, description: desc}, state) do
method handle_status (line 118) | def handle_status(%{status: status}, state) do
method handle_connected (line 124) | def handle_connected(consumer_info, state) do
class Publisher (line 143) | defmodule Publisher
method loop (line 146) | def loop(subject, attempt \\ 1, accepted \\ 0) do
FILE: test/command_test.exs
class Gnat.CommandTest (line 1) | defmodule Gnat.CommandTest
FILE: test/gnat/consumer_supervisor_test.exs
class Gnat.ConsumerSupervisorTest (line 1) | defmodule Gnat.ConsumerSupervisorTest
method start_service_supervisor (line 158) | defp start_service_supervisor(service_config) do
FILE: test/gnat/handshake_test.exs
class Gnat.HandshakeTest (line 1) | defmodule Gnat.HandshakeTest
FILE: test/gnat/parsec_property_test.exs
class Gnat.ParsecPropertyTest (line 1) | defmodule Gnat.ParsecPropertyTest
method random_chunk (line 81) | def random_chunk(binary), do: random_chunk(binary, [])
method random_chunk (line 82) | def random_chunk("", list), do: Enum.reverse(list)
method random_chunk (line 84) | def random_chunk(binary, list) do
FILE: test/gnat/parsec_test.exs
class Gnat.ParsecTest (line 1) | defmodule Gnat.ParsecTest
FILE: test/gnat_property_test.exs
class GnatPropertyTest (line 1) | defmodule GnatPropertyTest
FILE: test/gnat_test.exs
class GnatTest (line 1) | defmodule GnatTest
method spin_up_echo_server_on_topic (line 329) | defp spin_up_echo_server_on_topic(ready, gnat, topic) do
FILE: test/jetstream/api/consumer_doc_test.exs
class Gnat.Jetstream.API.ConsumerDocTest (line 1) | defmodule Gnat.Jetstream.API.ConsumerDocTest
FILE: test/jetstream/api/consumer_test.exs
class Gnat.Jetstream.API.ConsumerTest (line 1) | defmodule Gnat.Jetstream.API.ConsumerTest
FILE: test/jetstream/api/kv/entry_test.exs
class Gnat.Jetstream.API.KV.EntryTest (line 1) | defmodule Gnat.Jetstream.API.KV.EntryTest
FILE: test/jetstream/api/kv/watcher_test.exs
class Gnat.Jetstream.API.KV.WatcherTest (line 1) | defmodule Gnat.Jetstream.API.KV.WatcherTest
FILE: test/jetstream/api/kv_test.exs
class Gnat.Jetstream.API.KVTest (line 1) | defmodule Gnat.Jetstream.API.KVTest
FILE: test/jetstream/api/object_test.exs
class Gnat.Jetstream.API.ObjectTest (line 1) | defmodule Gnat.Jetstream.API.ObjectTest
method generate_big_file (line 240) | defp generate_big_file(tmp_dir) do
method put_filepath (line 256) | defp put_filepath(path, bucket, name) do
method stream_byte_size (line 261) | defp stream_byte_size(bucket) do
FILE: test/jetstream/api/stream_doc_test.exs
class Gnat.Jetstream.API.StreamDocTest (line 1) | defmodule Gnat.Jetstream.API.StreamDocTest
FILE: test/jetstream/api/stream_test.exs
class Gnat.Jetstream.API.StreamTest (line 1) | defmodule Gnat.Jetstream.API.StreamTest
FILE: test/jetstream/message_test.exs
class Gnat.Jetstream.MessageTest (line 1) | defmodule Gnat.Jetstream.MessageTest
FILE: test/jetstream/pager_test.exs
class Gnat.Jetstream.PagerTest (line 1) | defmodule Gnat.Jetstream.PagerTest
method create_stream (line 57) | defp create_stream(name) do
FILE: test/pull_consumer/batch_test.exs
class Gnat.Jetstream.PullConsumer.BatchTest (line 1) | defmodule Gnat.Jetstream.PullConsumer.BatchTest
class BatchPullConsumer (line 9) | defmodule BatchPullConsumer
method start_link (line 12) | def start_link(opts) do
method init (line 17) | def init(opts) do
method maybe_put (line 37) | defp maybe_put(conn_opts, key, opts) do
method handle_connected (line 45) | def handle_connected(consumer_info, state) do
method handle_message (line 51) | def handle_message(message, state) do
class NonAckBatchConsumer (line 60) | defmodule NonAckBatchConsumer
method start_link (line 63) | def start_link(opts) do
method init (line 68) | def init(opts) do
method handle_message (line 83) | def handle_message(%{body: "nack_me"}, state) do
method handle_message (line 88) | def handle_message(%{body: "term_me"}, state) do
method handle_message (line 93) | def handle_message(message, state) do
FILE: test/pull_consumer/connectivity_test.exs
class Gnat.Jetstream.PullConsumer.ConnectivityTest (line 1) | defmodule Gnat.Jetstream.PullConsumer.ConnectivityTest
method cleanup (line 168) | defp cleanup do
class ExamplePullConsumer (line 6) | defmodule ExamplePullConsumer
method start_link (line 9) | def start_link(opts) do
method init (line 14) | def init(opts) do
method handle_message (line 19) | def handle_message(%{topic: "ackable"}, state) do
method handle_message (line 23) | def handle_message(%{topic: "non-ackable", reply_to: reply_to}, state) do
method handle_message (line 34) | def handle_message(%{topic: "terminatable"}, state) do
method handle_message (line 38) | def handle_message(%{topic: "skippable"}, state) do
FILE: test/pull_consumer/ephemeral_test.exs
class Gnat.Jetstream.PullConsumer.EphemeralTest (line 1) | defmodule Gnat.Jetstream.PullConsumer.EphemeralTest
method cleanup (line 80) | defp cleanup do
class ExamplePullConsumer (line 6) | defmodule ExamplePullConsumer
method start_link (line 9) | def start_link(opts) do
method init (line 14) | def init(opts) do
method handle_connected (line 27) | def handle_connected(consumer_info, state) do
method handle_message (line 33) | def handle_message(message, state) do
FILE: test/pull_consumer/status_messages_test.exs
class Gnat.Jetstream.PullConsumer.StatusMessagesTest (line 1) | defmodule Gnat.Jetstream.PullConsumer.StatusMessagesTest
method drain_pulls (line 271) | defp drain_pulls(timeout) do
class ObservingConsumer (line 6) | defmodule ObservingConsumer
method start_link (line 9) | def start_link(opts) do
method init (line 14) | def init(opts) do
method handle_message (line 22) | def handle_message(message, state) do
method handle_status (line 28) | def handle_status(message, state) do
class SilentConsumer (line 34) | defmodule SilentConsumer
method start_link (line 37) | def start_link(opts) do
method init (line 42) | def init(opts) do
method handle_message (line 49) | def handle_message(message, test_pid) do
FILE: test/pull_consumer/using_macro_test.exs
class Gnat.Jetstream.PullConsumer.UsingMacroTest (line 1) | defmodule Gnat.Jetstream.PullConsumer.UsingMacroTest
class ExamplePullConsumer (line 4) | defmodule ExamplePullConsumer
method start_link (line 7) | def start_link(_) do
method init (line 12) | def init([]) do
method handle_message (line 17) | def handle_message(%{}, state) do
FILE: test/support/conn_case.ex
class Gnat.Jetstream.ConnCase (line 1) | defmodule Gnat.Jetstream.ConnCase
method module_tags (line 18) | defp module_tags(opts) do
method add_module_tag (line 24) | defp add_module_tag(tags, {:min_server_version, min_version}) do
method add_module_tag (line 32) | defp add_module_tag(tags, _opt), do: tags
method get_server_version (line 34) | defp get_server_version(conn) do
method server_version_incompatible? (line 38) | defp server_version_incompatible?(min_version) do
FILE: test/support/generators.ex
class Gnat.Generators (line 1) | defmodule Gnat.Generators
method alphanumeric_char (line 5) | def alphanumeric_char do
method alphanumeric_space_char (line 9) | def alphanumeric_space_char do
method numeric_char (line 13) | def numeric_char do
method delimiter (line 18) | def delimiter, do: let(chunks <- non_empty(list(delimiter_char())), do...
method delimiter_char (line 20) | def delimiter_char, do: union([" ", "\t"])
method error (line 22) | def error do
method host_port (line 28) | def host_port do
method info (line 35) | def info do
method info_options (line 41) | def info_options do
method ok (line 62) | def ok, do: %{binary: "+OK\r\n"}
method ping (line 64) | def ping, do: %{binary: "PING\r\n"}
method pong (line 66) | def pong, do: %{binary: "PONG\r\n"}
method message (line 70) | def message, do: sized(size, message(size))
method message (line 71) | def message(size), do: union([message_without_reply(size), message_wit...
method message_with_reply (line 73) | def message_with_reply(size) do
method message_without_reply (line 91) | def message_without_reply(size) do
method payload (line 108) | def payload(size), do: binary(size)
method protocol_message (line 110) | def protocol_message do
method sid (line 116) | def sid, do: non_neg_integer()
method subject (line 118) | def subject do
method subject_chunks (line 122) | def subject_chunks do
method reply_to (line 129) | def reply_to, do: subject()
FILE: test/test_helper.exs
class RpcEndpoint (line 35) | defmodule RpcEndpoint
method init (line 36) | def init do
method loop (line 42) | def loop(pid) do
class ExampleService (line 53) | defmodule ExampleService
method request (line 68) | def request(_, _, _) do
method error (line 72) | def error(_msg, "oops") do
class ExampleServer (line 77) | defmodule ExampleServer
method request (line 80) | def request(%{topic: "example.good", body: body}) do
method request (line 84) | def request(%{topic: "example.error"}) do
method request (line 88) | def request(%{topic: "example.raise"}) do
method error (line 92) | def error(_msg, "oops") do
method error (line 96) | def error(_msg, %RuntimeError{message: "oops"}) do
method error (line 100) | def error(msg, other) do
class CheckForExpectedNatsServers (line 136) | defmodule CheckForExpectedNatsServers
method check (line 137) | def check(tags) do
method check_for_default (line 142) | def check_for_default do
method check_for_tag (line 157) | def check_for_tag(:multi_server) do
method check_for_tag (line 242) | def check_for_tag(_), do: :ok
Condensed preview — 85 files, each showing path, character count, and a content snippet. Download the .json file or copy for the full structured content (416K chars).
[
{
"path": ".dialyzer_ignore.exs",
"chars": 4,
"preview": "[\n]\n"
},
{
"path": ".formatter.exs",
"chars": 67,
"preview": "[\n inputs: [\"{.formatter,mix}.exs\", \"{lib,test}/**/*.{ex,exs}\"]\n]\n"
},
{
"path": ".github/workflows/CI.yml",
"chars": 2503,
"preview": "name: CI\n\non:\n push:\n branches: [main]\n pull_request:\n branches: [main]\n schedule:\n - cron: \"0 0 1 */1 *\"\n\np"
},
{
"path": ".gitignore",
"chars": 824,
"preview": "# The directory Mix will write compiled artifacts to.\n/_build\n\n# If you run \"mix test --cover\", coverage assets end up h"
},
{
"path": "CHANGELOG.md",
"chars": 6313,
"preview": "# Changelog\n\n## 1.14\n\n* Add `PullConsumer.handle_connected/2` optional callback to get consumer info\n* Add `PullConsumer"
},
{
"path": "LICENSE.txt",
"chars": 1052,
"preview": "Copyright 2017 Michael Ries\n\nPermission is hereby granted, free of charge, to any person obtaining a copy of this softwa"
},
{
"path": "MAINTAINERS.md",
"chars": 327,
"preview": "# Maintainers\n\nMaintainership is on a per project basis.\n\n### Maintainers\n - Colin Sullivan <colin@nats.io> [@ColinSull"
},
{
"path": "README.md",
"chars": 7480,
"preview": "[](https://hex.pm/packages/gnat)\n[![hex.pm](https://img.shields.io/hex"
},
{
"path": "bench/client.exs",
"chars": 3267,
"preview": "Application.put_env(:client, :num_connections, 4)\nnum_requesters = 16\nrequests_per_requester = 500\n\ndefmodule Client do\n"
},
{
"path": "bench/kv_consume.exs",
"chars": 8420,
"preview": "# bench/kv_consume.exs\n#\n# Compares three approaches for consuming all messages from a KV bucket into ETS:\n#\n# 1. Page"
},
{
"path": "bench/parse.exs",
"chars": 666,
"preview": "msg1024 = :crypto.strong_rand_bytes(1024)\nmsg128 = :crypto.strong_rand_bytes(128)\nmsg16 = :crypto.strong_rand_bytes(1"
},
{
"path": "bench/publish.exs",
"chars": 418,
"preview": "inputs = %{\n \"16 byte\" => :crypto.strong_rand_bytes(16),\n \"128 byte\" => :crypto.strong_rand_bytes(128),\n \"1024 byte\" "
},
{
"path": "bench/request.exs",
"chars": 937,
"preview": "defmodule EchoServer do\n def run(gnat) do\n spawn(fn -> init(gnat) end)\n end\n\n def init(gnat) do\n Gnat.sub(gnat,"
},
{
"path": "bench/request_multi.exs",
"chars": 1067,
"preview": "defmodule EchoServer do\n def run(gnat) do\n spawn(fn -> init(gnat) end)\n end\n\n def init(gnat) do\n Gnat.sub(gnat,"
},
{
"path": "bench/server.exs",
"chars": 745,
"preview": "num_connections = 4\nnum_subscribers = 4\n\nEnum.each(0..(num_connections - 1), fn(i) ->\n name = :\"gnat#{i}\"\n {:ok, _pid}"
},
{
"path": "bench/service_bench.exs",
"chars": 1599,
"preview": "defmodule EchoService do\n use Gnat.Services.Server\n\n def request(%{body: body}, \"echo\", _group) do\n {:reply, body}\n"
},
{
"path": "dependencies.md",
"chars": 1020,
"preview": "# Project Dependencies\n\nThis is a list of dependencies that will be pulled into your project when you use this library. "
},
{
"path": "docs/js/guides/broadway.md",
"chars": 4856,
"preview": "# Using Broadway with Jetstream\n\nBroadway is a library which allows building concurrent and multi-stage data ingestion a"
},
{
"path": "docs/js/guides/managing.md",
"chars": 492,
"preview": "# Managing Streams and Consumers\n\nJetstream provides a JSON API for managing streams and consumers.\nThis library exposes"
},
{
"path": "docs/js/guides/push_based_consumer.md",
"chars": 2053,
"preview": "# Push based consumer\n\n```elixir\n# Start a nats server with jetstream enabled and default configs\n# Now run the followin"
},
{
"path": "docs/js/introduction/getting_started.md",
"chars": 5964,
"preview": "# Getting Started\n\nIn this guide, we're going to learn how to install Jetstream in your project and start consuming\nmess"
},
{
"path": "docs/js/introduction/overview.md",
"chars": 802,
"preview": "# Overview\n\n[Jetstream](https://docs.nats.io/nats-concepts/jetstream) is a distributed persistence system\nbuilt-in to [N"
},
{
"path": "lib/gnat/command.ex",
"chars": 1923,
"preview": "defmodule Gnat.Command do\n @moduledoc false\n\n @newline \"\\r\\n\"\n @hpub \"HPUB\"\n @pub \"PUB\"\n @sub \"SUB\"\n @unsub \"UNSUB"
},
{
"path": "lib/gnat/connection_supervisor.ex",
"chars": 3560,
"preview": "defmodule Gnat.ConnectionSupervisor do\n use GenServer\n require Logger\n\n @moduledoc \"\"\"\n A process that can supervise"
},
{
"path": "lib/gnat/consumer_supervisor.ex",
"chars": 9538,
"preview": "defmodule Gnat.ConsumerSupervisor do\n use GenServer\n require Logger\n alias Gnat.Services.Service\n\n @moduledoc \"\"\"\n "
},
{
"path": "lib/gnat/handshake.ex",
"chars": 3854,
"preview": "defmodule Gnat.Handshake do\n @moduledoc false\n alias Gnat.Parsec\n\n @doc \"\"\"\n This function handles all of the variat"
},
{
"path": "lib/gnat/jetstream/api/consumer.ex",
"chars": 22467,
"preview": "defmodule Gnat.Jetstream.API.Consumer do\n @moduledoc \"\"\"\n A module representing a NATS JetStream Consumer.\n\n Learn mo"
},
{
"path": "lib/gnat/jetstream/api/kv/entry.ex",
"chars": 5081,
"preview": "defmodule Gnat.Jetstream.API.KV.Entry do\n @moduledoc \"\"\"\n A parsed view of a single message from a Key/Value bucket's "
},
{
"path": "lib/gnat/jetstream/api/kv/watcher.ex",
"chars": 3666,
"preview": "defmodule Gnat.Jetstream.API.KV.Watcher do\n @moduledoc \"\"\"\n The watcher server establishes a subscription to the chang"
},
{
"path": "lib/gnat/jetstream/api/kv.ex",
"chars": 12786,
"preview": "defmodule Gnat.Jetstream.API.KV do\n @moduledoc \"\"\"\n API for interacting with the Key/Value store functionality in Nats"
},
{
"path": "lib/gnat/jetstream/api/message.ex",
"chars": 2927,
"preview": "defmodule Gnat.Jetstream.API.Message do\n @moduledoc \"\"\"\n This module provides a way to parse the `reply_to` received b"
},
{
"path": "lib/gnat/jetstream/api/object/meta.ex",
"chars": 945,
"preview": "defmodule Gnat.Jetstream.API.Object.Meta do\n @enforce_keys [:bucket, :chunks, :digest, :name, :nuid, :size]\n defstruct"
},
{
"path": "lib/gnat/jetstream/api/object.ex",
"chars": 11015,
"preview": "defmodule Gnat.Jetstream.API.Object do\n @moduledoc \"\"\"\n API for interacting with the JetStream Object Store\n\n Learn m"
},
{
"path": "lib/gnat/jetstream/api/stream.ex",
"chars": 21841,
"preview": "defmodule Gnat.Jetstream.API.Stream do\n @moduledoc \"\"\"\n A module representing a NATS JetStream Stream.\n\n Learn more a"
},
{
"path": "lib/gnat/jetstream/api/util.ex",
"chars": 1334,
"preview": "defmodule Gnat.Jetstream.API.Util do\n @moduledoc false\n\n @default_inbox_prefix \"_INBOX.\"\n\n def request(conn, topic, p"
},
{
"path": "lib/gnat/jetstream/jetstream.ex",
"chars": 2354,
"preview": "defmodule Gnat.Jetstream do\n @moduledoc \"\"\"\n Provides functions for interacting with a [NATS Jetstream](https://github"
},
{
"path": "lib/gnat/jetstream/pager.ex",
"chars": 5102,
"preview": "defmodule Gnat.Jetstream.Pager do\n @moduledoc \"\"\"\n Page through all the messages in a stream\n\n This module provides a"
},
{
"path": "lib/gnat/jetstream/pull_consumer/connection_options.ex",
"chars": 2703,
"preview": "defmodule Gnat.Jetstream.PullConsumer.ConnectionOptions do\n @moduledoc false\n\n @default_retry_timeout 1000\n @default_"
},
{
"path": "lib/gnat/jetstream/pull_consumer/server.ex",
"chars": 17400,
"preview": "defmodule Gnat.Jetstream.PullConsumer.Server do\n @moduledoc false\n\n require Logger\n\n use Connection\n\n alias Gnat.Jet"
},
{
"path": "lib/gnat/jetstream/pull_consumer.ex",
"chars": 16559,
"preview": "defmodule Gnat.Jetstream.PullConsumer do\n @moduledoc \"\"\"\n A behaviour which provides the NATS JetStream Pull Consumer "
},
{
"path": "lib/gnat/parsec.ex",
"chars": 5537,
"preview": "defmodule Gnat.Parsec do\n @moduledoc false\n defstruct partial: nil\n\n import NimbleParsec\n\n subject = ascii_string([?"
},
{
"path": "lib/gnat/server.ex",
"chars": 3234,
"preview": "defmodule Gnat.Server do\n require Logger\n\n @moduledoc \"\"\"\n A behavior for acting as a server for nats messages.\n\n Yo"
},
{
"path": "lib/gnat/services/server.ex",
"chars": 6837,
"preview": "defmodule Gnat.Services.Server do\n require Logger\n alias Gnat.Services.{Service, ServiceResponder}\n\n @moduledoc \"\"\"\n "
},
{
"path": "lib/gnat/services/service.ex",
"chars": 6539,
"preview": "defmodule Gnat.Services.Service do\n @moduledoc false\n\n @subscription_subject \"$SRV.>\"\n # required default, see https:"
},
{
"path": "lib/gnat/services/service_responder.ex",
"chars": 1586,
"preview": "defmodule Gnat.Services.ServiceResponder do\n @moduledoc false\n\n require Logger\n alias Gnat.Services.Service\n\n @op_pi"
},
{
"path": "lib/gnat/services/wire_protocol.ex",
"chars": 2208,
"preview": "defmodule Gnat.Services.WireProtocol do\n @moduledoc false\n\n defmodule InfoResponse do\n @moduledoc false\n @info_r"
},
{
"path": "lib/gnat.ex",
"chars": 27470,
"preview": "# State transitions:\n# :waiting_for_message => receive PING, send PONG => :waiting_for_message\n# :waiting_for_message "
},
{
"path": "mix.exs",
"chars": 2411,
"preview": "defmodule Gnat.Mixfile do\n use Mix.Project\n\n @source_url \"https://github.com/nats-io/nats.ex\"\n @version \"1.14.0\"\n\n d"
},
{
"path": "scripts/cluster/cluster.sh",
"chars": 3043,
"preview": "#!/usr/bin/env bash\n# Minimal control script for a 3-node local nats cluster used for\n# manually exercising PullConsumer"
},
{
"path": "scripts/cluster/driver.exs",
"chars": 5490,
"preview": "# Manual failover driver for the 3-node cluster under scripts/cluster/.\n#\n# Usage (from repo root, after `scripts/cluste"
},
{
"path": "scripts/cluster/n1.conf",
"chars": 239,
"preview": "port: 4223\nhttp_port: 8223\nserver_name: n1\n\njetstream {\n store_dir: \"./scripts/cluster/data/n1\"\n}\n\ncluster {\n name: fa"
},
{
"path": "scripts/cluster/n2.conf",
"chars": 239,
"preview": "port: 4224\nhttp_port: 8224\nserver_name: n2\n\njetstream {\n store_dir: \"./scripts/cluster/data/n2\"\n}\n\ncluster {\n name: fa"
},
{
"path": "scripts/cluster/n3.conf",
"chars": 239,
"preview": "port: 4225\nhttp_port: 8225\nserver_name: n3\n\njetstream {\n store_dir: \"./scripts/cluster/data/n3\"\n}\n\ncluster {\n name: fa"
},
{
"path": "test/command_test.exs",
"chars": 1182,
"preview": "defmodule Gnat.CommandTest do\n use ExUnit.Case, async: true\n alias Gnat.Command\n\n test \"formatting a simple pub messa"
},
{
"path": "test/fixtures/ca.pem",
"chars": 1223,
"preview": "-----BEGIN CERTIFICATE-----\nMIIDXDCCAkQCCQDI2Vsry8+BDDANBgkqhkiG9w0BAQsFADBwMQswCQYDVQQGEwJV\nUzELMAkGA1UECAwCQ0ExEDAOBgN"
},
{
"path": "test/fixtures/client-cert.pem",
"chars": 1191,
"preview": "-----BEGIN CERTIFICATE-----\nMIIDQjCCAiqgAwIBAgIJAJCSLX9jr5WzMA0GCSqGSIb3DQEBBQUAMHAxCzAJBgNV\nBAYTAlVTMQswCQYDVQQIDAJDQTE"
},
{
"path": "test/fixtures/client-key.pem",
"chars": 1707,
"preview": "-----BEGIN PRIVATE KEY-----\nMIIEvwIBADANBgkqhkiG9w0BAQEFAASCBKkwggSlAgEAAoIBAQCsnD6dO3oSVoV4\nyt+c/Ax+XvJPIjNGgThT16clj9f"
},
{
"path": "test/fixtures/nkey_config",
"chars": 104,
"preview": "authorization: {\n users: [\n { nkey: UBSUDO5PNPFR72YUCWWSN4ADPIEU3WESNZ35S3VZAWERPXCZSQTDH7SS }\n ]\n}"
},
{
"path": "test/fixtures/nkey_seed",
"chars": 58,
"preview": "SUAMH3IDGSDQ2AVZFOWYAKNA7R2FXIZZSQ3BQMA5QNJRYP3ABIKDDP5DBA"
},
{
"path": "test/fixtures/server-cert.pem",
"chars": 1183,
"preview": "-----BEGIN CERTIFICATE-----\nMIIDPTCCAiWgAwIBAgIJAJCSLX9jr5W7MA0GCSqGSIb3DQEBBQUAMHAxCzAJBgNV\nBAYTAlVTMQswCQYDVQQIDAJDQTE"
},
{
"path": "test/fixtures/server-key.pem",
"chars": 1699,
"preview": "-----BEGIN PRIVATE KEY-----\nMIIEuwIBADANBgkqhkiG9w0BAQEFAASCBKUwggShAgEAAoIBAQDm+0dlzcmiLa+L\nzdVqeVQ8B1/rWnErK+VvvjH7FmV"
},
{
"path": "test/gnat/consumer_supervisor_test.exs",
"chars": 5339,
"preview": "defmodule Gnat.ConsumerSupervisorTest do\n alias Gnat.ConsumerSupervisor\n use ExUnit.Case, async: true\n\n # these reque"
},
{
"path": "test/gnat/handshake_test.exs",
"chars": 3384,
"preview": "defmodule Gnat.HandshakeTest do\n use ExUnit.Case, async: true\n alias Gnat.Handshake\n\n describe \"negotiate_settings/2\""
},
{
"path": "test/gnat/parsec_property_test.exs",
"chars": 2911,
"preview": "defmodule Gnat.ParsecPropertyTest do\n use ExUnit.Case, async: true\n use PropCheck\n import Gnat.Generators, only: [mes"
},
{
"path": "test/gnat/parsec_test.exs",
"chars": 6385,
"preview": "defmodule Gnat.ParsecTest do\n use ExUnit.Case, async: true\n alias Gnat.Parsec\n\n test \"parsing a complete message\" do\n"
},
{
"path": "test/gnat_property_test.exs",
"chars": 1223,
"preview": "defmodule GnatPropertyTest do\n use ExUnit.Case, async: true\n use PropCheck\n import Gnat.Generators, only: [message: 0"
},
{
"path": "test/gnat_test.exs",
"chars": 12292,
"preview": "defmodule GnatTest do\n use ExUnit.Case, async: true\n doctest Gnat\n\n setup context do\n CheckForExpectedNatsServers."
},
{
"path": "test/jetstream/api/consumer_doc_test.exs",
"chars": 150,
"preview": "defmodule Gnat.Jetstream.API.ConsumerDocTest do\n use Gnat.Jetstream.ConnCase\n @moduletag with_gnat: :gnat\n doctest Gn"
},
{
"path": "test/jetstream/api/consumer_test.exs",
"chars": 8942,
"preview": "defmodule Gnat.Jetstream.API.ConsumerTest do\n use Gnat.Jetstream.ConnCase\n alias Gnat.Jetstream.API.{Consumer, Stream}"
},
{
"path": "test/jetstream/api/kv/entry_test.exs",
"chars": 3155,
"preview": "defmodule Gnat.Jetstream.API.KV.EntryTest do\n use ExUnit.Case, async: true\n\n alias Gnat.Jetstream.API.KV.Entry\n\n @buc"
},
{
"path": "test/jetstream/api/kv/watcher_test.exs",
"chars": 1850,
"preview": "defmodule Gnat.Jetstream.API.KV.WatcherTest do\n use Gnat.Jetstream.ConnCase, min_server_version: \"2.6.2\"\n\n alias Gnat."
},
{
"path": "test/jetstream/api/kv_test.exs",
"chars": 10858,
"preview": "defmodule Gnat.Jetstream.API.KVTest do\n use Gnat.Jetstream.ConnCase, min_server_version: \"2.6.2\"\n alias Gnat.Jetstream"
},
{
"path": "test/jetstream/api/object_test.exs",
"chars": 9437,
"preview": "defmodule Gnat.Jetstream.API.ObjectTest do\n use Gnat.Jetstream.ConnCase, min_server_version: \"2.6.2\"\n alias Gnat.Jetst"
},
{
"path": "test/jetstream/api/stream_doc_test.exs",
"chars": 146,
"preview": "defmodule Gnat.Jetstream.API.StreamDocTest do\n use Gnat.Jetstream.ConnCase\n @moduletag with_gnat: :gnat\n doctest Gnat"
},
{
"path": "test/jetstream/api/stream_test.exs",
"chars": 11767,
"preview": "defmodule Gnat.Jetstream.API.StreamTest do\n use Gnat.Jetstream.ConnCase\n alias Gnat.Jetstream.API.Stream\n\n @moduletag"
},
{
"path": "test/jetstream/message_test.exs",
"chars": 1266,
"preview": "defmodule Gnat.Jetstream.MessageTest do\n use ExUnit.Case, async: true\n\n test \"message metadata without domain\" do\n "
},
{
"path": "test/jetstream/pager_test.exs",
"chars": 1585,
"preview": "defmodule Gnat.Jetstream.PagerTest do\n use Gnat.Jetstream.ConnCase\n alias Gnat.Jetstream.Pager\n alias Gnat.Jetstream."
},
{
"path": "test/pull_consumer/batch_test.exs",
"chars": 14548,
"preview": "defmodule Gnat.Jetstream.PullConsumer.BatchTest do\n use Gnat.Jetstream.ConnCase\n\n alias Gnat.Jetstream.API.{Consumer, "
},
{
"path": "test/pull_consumer/connectivity_test.exs",
"chars": 5188,
"preview": "defmodule Gnat.Jetstream.PullConsumer.ConnectivityTest do\n use Gnat.Jetstream.ConnCase\n\n alias Gnat.Jetstream.API.{Con"
},
{
"path": "test/pull_consumer/ephemeral_test.exs",
"chars": 2216,
"preview": "defmodule Gnat.Jetstream.PullConsumer.EphemeralTest do\n use Gnat.Jetstream.ConnCase\n\n alias Gnat.Jetstream.API.{Consum"
},
{
"path": "test/pull_consumer/status_messages_test.exs",
"chars": 8982,
"preview": "defmodule Gnat.Jetstream.PullConsumer.StatusMessagesTest do\n use Gnat.Jetstream.ConnCase\n\n alias Gnat.Jetstream.API.{C"
},
{
"path": "test/pull_consumer/using_macro_test.exs",
"chars": 822,
"preview": "defmodule Gnat.Jetstream.PullConsumer.UsingMacroTest do\n use Gnat.Jetstream.ConnCase\n\n defmodule ExamplePullConsumer d"
},
{
"path": "test/support/conn_case.ex",
"chars": 1470,
"preview": "defmodule Gnat.Jetstream.ConnCase do\n @moduledoc \"\"\"\n This module defines the test case to be used by tests that requi"
},
{
"path": "test/support/generators.ex",
"chars": 3762,
"preview": "defmodule Gnat.Generators do\n use PropCheck\n\n # Character classes useful for generating text\n def alphanumeric_char d"
},
{
"path": "test/test_helper.exs",
"chars": 6384,
"preview": "ExUnit.configure(exclude: [:pending, :property, :multi_server, :message_ttl])\n\nExUnit.start()\n\n# set assert_receive defa"
}
]
About this extraction
This page contains the full source code of the nats-io/nats.ex GitHub repository, extracted and formatted as plain text for AI agents and large language models (LLMs). The extraction includes 85 files (384.2 KB), approximately 109.9k tokens, and a symbol index with 493 extracted functions, classes, methods, constants, and types. Use this with OpenClaw, Claude, ChatGPT, Cursor, Windsurf, or any other AI tool that accepts text input. You can copy the full output to your clipboard or download it as a .txt file.
Extracted by GitExtract — free GitHub repo to text converter for AI. Built by Nikandr Surkov.