Repository: nats-io/nats.ex Branch: main Commit: b6d2f37e8901 Files: 85 Total size: 384.2 KB Directory structure: gitextract_y72cyuck/ ├── .dialyzer_ignore.exs ├── .formatter.exs ├── .github/ │ └── workflows/ │ └── CI.yml ├── .gitignore ├── CHANGELOG.md ├── LICENSE.txt ├── MAINTAINERS.md ├── README.md ├── bench/ │ ├── client.exs │ ├── kv_consume.exs │ ├── parse.exs │ ├── publish.exs │ ├── request.exs │ ├── request_multi.exs │ ├── server.exs │ └── service_bench.exs ├── dependencies.md ├── docs/ │ └── js/ │ ├── guides/ │ │ ├── broadway.md │ │ ├── managing.md │ │ └── push_based_consumer.md │ └── introduction/ │ ├── getting_started.md │ └── overview.md ├── lib/ │ ├── gnat/ │ │ ├── command.ex │ │ ├── connection_supervisor.ex │ │ ├── consumer_supervisor.ex │ │ ├── handshake.ex │ │ ├── jetstream/ │ │ │ ├── api/ │ │ │ │ ├── consumer.ex │ │ │ │ ├── kv/ │ │ │ │ │ ├── entry.ex │ │ │ │ │ └── watcher.ex │ │ │ │ ├── kv.ex │ │ │ │ ├── message.ex │ │ │ │ ├── object/ │ │ │ │ │ └── meta.ex │ │ │ │ ├── object.ex │ │ │ │ ├── stream.ex │ │ │ │ └── util.ex │ │ │ ├── jetstream.ex │ │ │ ├── pager.ex │ │ │ ├── pull_consumer/ │ │ │ │ ├── connection_options.ex │ │ │ │ └── server.ex │ │ │ └── pull_consumer.ex │ │ ├── parsec.ex │ │ ├── server.ex │ │ └── services/ │ │ ├── server.ex │ │ ├── service.ex │ │ ├── service_responder.ex │ │ └── wire_protocol.ex │ └── gnat.ex ├── mix.exs ├── scripts/ │ └── cluster/ │ ├── cluster.sh │ ├── driver.exs │ ├── n1.conf │ ├── n2.conf │ └── n3.conf └── test/ ├── command_test.exs ├── fixtures/ │ ├── ca.pem │ ├── client-cert.pem │ ├── client-key.pem │ ├── nkey_config │ ├── nkey_seed │ ├── server-cert.pem │ └── server-key.pem ├── gnat/ │ ├── consumer_supervisor_test.exs │ ├── handshake_test.exs │ ├── parsec_property_test.exs │ └── parsec_test.exs ├── gnat_property_test.exs ├── gnat_test.exs ├── jetstream/ │ ├── api/ │ │ ├── consumer_doc_test.exs │ │ ├── consumer_test.exs │ │ ├── kv/ │ │ │ ├── entry_test.exs │ │ │ └── watcher_test.exs │ │ ├── kv_test.exs │ │ ├── object_test.exs │ │ ├── stream_doc_test.exs │ │ └── stream_test.exs │ ├── message_test.exs │ └── pager_test.exs ├── pull_consumer/ │ ├── batch_test.exs │ ├── connectivity_test.exs │ ├── ephemeral_test.exs │ ├── status_messages_test.exs │ └── using_macro_test.exs ├── support/ │ ├── conn_case.ex │ └── generators.ex └── test_helper.exs ================================================ FILE CONTENTS ================================================ ================================================ FILE: .dialyzer_ignore.exs ================================================ [ ] ================================================ FILE: .formatter.exs ================================================ [ inputs: ["{.formatter,mix}.exs", "{lib,test}/**/*.{ex,exs}"] ] ================================================ FILE: .github/workflows/CI.yml ================================================ name: CI on: push: branches: [main] pull_request: branches: [main] schedule: - cron: "0 0 1 */1 *" permissions: contents: read jobs: test: name: Elixir ${{ matrix.pair.elixir }} / OTP ${{ matrix.pair.otp }} / NATS ${{ matrix.pair.nats }} runs-on: ubuntu-latest strategy: matrix: pair: - otp: "25" elixir: "1.14" nats: "2.10.0" - otp: "26" elixir: "1.16" nats: "2.10.24" - otp: "27" elixir: "1.17" nats: "2.10.24" - otp: "27" elixir: "1.18" nats: "2.10.24" - otp: "28" elixir: "1.18" nats: "2.11.11" - otp: "28" elixir: "1.18" nats: "2.10.24" # the main pair handles things like format checking and running dialyzer - main: true otp: "28" elixir: "1.19" nats: "2.10.24" env: MIX_ENV: test steps: - name: Checkout uses: actions/checkout@v4 - name: Set up Elixir uses: erlef/setup-beam@v1 with: elixir-version: ${{ matrix.pair.elixir }} otp-version: ${{ matrix.pair.otp }} - name: Start NATS Jetstream run: docker run --rm -d --network host nats:${{ matrix.pair.nats }} -js - name: Restore deps cache uses: actions/cache@v4 with: path: | deps _build key: deps-${{ matrix.pair.otp }}-${{ matrix.pair.elixir }}-${{ hashFiles('mix.lock') }} - name: Install package dependencies run: mix deps.get - name: Check for valid formatting if: matrix.pair.main run: mix format --check-formatted - name: Run unit tests run: mix test --color - name: Run NATS 2.11 specific tests if: ${{ matrix.pair.nats == '2.11.11' }} run: mix test --only message_ttl - name: Cache Dialyzer PLTs if: matrix.pair.main uses: actions/cache@v4 with: path: | priv/plts/*.plt priv/plts/*.plt.hash key: dialyzer-${{ matrix.pair.otp }}-${{ matrix.pair.elixir }}-${{ hashFiles('mix.lock') }} restore-keys: | dialyzer-${{ matrix.pair.otp }}-${{ matrix.pair.elixir }}- - name: Run Dialyzer if: matrix.pair.main env: MIX_ENV: dev run: | mix dialyzer ================================================ FILE: .gitignore ================================================ # The directory Mix will write compiled artifacts to. /_build # If you run "mix test --cover", coverage assets end up here. /cover # The directory Mix downloads your dependencies sources to. /deps /tmp # Where 3rd-party dependencies like ExDoc output generated docs. /doc # Dialyzer PLT files priv/plts/ # Ignore .fetch files in case you like to edit your project deps locally. /.fetch # If the VM crashes, it generates a dump, let's ignore it too. erl_crash.dump # Also ignore archive artifacts (built via "mix archive.build"). *.ez # Do not commit the counterexamples stored during property testing /test/counter_examples # ASDF file to for specifying which erlang/elixir version to use for local development .tool-versions # Ignore Cluster details scripts/cluster/data scripts/cluster/logs scripts/cluster/pids ================================================ FILE: CHANGELOG.md ================================================ # Changelog ## 1.14 * Add `PullConsumer.handle_connected/2` optional callback to get consumer info * Add `PullConsumer.handle_status/2` optional callback to observe status messages * Added support for `batch_size` option in PullConsumer options to pull messages and acknowledge them in batches. * Add `Gnat.Jetstream.API.KV.Entry` with `from_message/2` for parsing a raw NATS message from a KV bucket's underlying stream into a structured entry (operation, key, value, revision, created, delta). Intended to be shared between the built-in `KV.Watcher` and user-supplied `PullConsumer` implementations (e.g. caches that need to detect when they are caught up with the stream). Returns `:ignore` for messages that are not KV records. * `KV.Watcher` now uses `KV.Entry` internally; its public callback API is unchanged. The push consumer it creates now enables server-driven flow control and a 5s idle heartbeat (matching nats.go's ordered-consumer defaults), so slow handlers apply backpressure instead of being dropped as slow consumers. * **Behavior change (bugfix):** `PullConsumer` no longer forwards JetStream informational status messages (e.g. `100` idle heartbeat, `409` leadership change) to `c:handle_message/2`. These are not stream records and cannot be acked. In single-message mode the consumer now drops them and re-issues a pull request. * Add an optional `c:handle_status/2` callback to `Gnat.Jetstream.PullConsumer` for users who want to observe status messages (e.g. log on `409`). ## 1.11 * Allow clients to force authentication without server auth_required by @mmmries in #205 * Support ephemeral consumers and auto-cleanup consumers in the PullConsumer module by @mmmries in #202 * Implement regularly scheduled PING/PONG health check by @mmmries in #200 * Add Erlang 28 with Elixir 1.18 to the build matrix by @davydog187 in #201 * Use Pager module for KV.contents by @mmmries in #198 * Hint `:timeout` option in KV's typespec by @rixmann in #193 ## 1.10 * Clarify authentication setup during test by @davydog187 in #187 * Test on Elixir 1.18 and NATS 2.10.24 by @davydog187 in #188 * Gnat.Jetstream.API.KV.info/3 by @davydog187 in #189 * Remove function_exported? check for Keyword.validate!/2 by @davydog187 in #186 * Tiny optimization to KV.list_buckets/1 by @davydog187 in #185 * make KV-watcher emit :key_added events when the message has a header by @rixmann in #191 * add :compression to stream attributes by @rixmann in #192 * fix: unknown field domain in Stream.create (#194) by @c0deaddict * feat: add jetstream message metadata helper (#197) by @c0deaddict * fix: deliver policy (#196) by @c0deaddict ## 1.9 * Housecleaning by @mmmries in #176 * switch to charlist sigils * update to newest nkeys * require elixir 1.14 and erlang 25+ * Fix incorrect useage of charlist by @davydog187 in #179 * Soft deprecate is_kv_bucket_stream?/1 in favor of kv_bucket_stream?/1 by @davydog187 in #183 * Clean up examples in KV by @davydog187 in #181 * Document options for Gnat.Jetstream.API.KV by @davydog187 in #180 ## 1.8 * Integrated the jetstream functionality into this client directly https://github.com/nats-io/nats.ex/pull/146 * Add ability to list KV buckets https://github.com/nats-io/nats.ex/pull/152 * Improve CI Reliability https://github.com/nats-io/nats.ex/pull/154 * Bugfix to treat no streams as an empty list rather than a null https://github.com/nats-io/nats.ex/pull/155 * Added supported for `allow_direct` and `mirror_direct` attributes of streams https://github.com/nats-io/nats.ex/pull/161 * Added support for `discard_new_per_subject` attribute of streams https://github.com/nats-io/nats.ex/pull/163 * Added support for `Object.list_buckets` https://github.com/nats-io/nats.ex/pull/169 ## 1.7 * Added support for the NATS [services API](https://github.com/nats-io/nats.go/blob/main/micro/README.md), letting developers participate in service discovery and stats https://github.com/nats-io/nats.ex/pull/141 * A bugfix to remove the queue_group from a service config and some optimization for the services API https://github.com/nats-io/nats.ex/pull/145 ## 1.6 * added the `no_responders` behavior https://github.com/nats-io/nats.ex/pull/137 ## 1.5 * add the `inbox_prefix` option https://github.com/nats-io/nats.ex/pull/121 * add the `Gnat.server_info/1` function https://github.com/nats-io/nats.ex/pull/124 * fix header parsing issue https://github.com/nats-io/nats.ex/pull/125 ## 1.4 * add the `Gnat.request_multi/4` function https://github.com/nats-io/nats.ex/pull/120 * add elixir 1.13 to the test matrix ## 1.3 * adding support for sending and receiving headers https://github.com/nats-io/nats.ex/pull/116 ## 1.2 * `Gnat.Server` behaviour with support in the `ConsumerSupervisor` https://github.com/nats-io/nats.ex/compare/1b1adc85e4b28231218ef87c7fc3445fce854377...b24a7e14325b51fbb93fde7e3d891d18b4fa8afb * avoid logging sensitive credentials https://github.com/nats-io/nats.ex/pull/105 * deprecate Gnat.ping, improved typespecs https://github.com/nats-io/nats.ex/pull/103 * relax the version constraint on nimble_parsec https://github.com/nats-io/nats.ex/issues/112 ## 1.1 * add support for nkeys and NGS https://github.com/nats-io/nats.ex/pull/101 * Fix supervisor ConsumerSuperivsor crash https://github.com/nats-io/nats.ex/pull/96 ## 1.0 * Make supervisors officially supported https://github.com/nats-io/nats.ex/pull/96 ## 0.7.0 * update to telemetry 0.4 https://github.com/nats-io/nats.ex/pull/86 and https://github.com/nats-io/nats.ex/pull/87 * support for token authentication https://github.com/nats-io/nats.ex/pull/92 * support elixir 1.9 https://github.com/nats-io/nats.ex/pull/93 ## 0.6.0 * Dropped support for Erlang < 19 and Elixir <= 1.5 * Added Telemetry to the project (thanks @rubysolo) * Switched to nimble_parsec for parsing * Updated benchmarking/performance information. We can now do 170k requests per second on a 16-core server. * Fixed a bug around re-subscribing for the `ConsumerSupervisor` * Pass `sid` when delivering message (thanks @entone) * Documentation fixes from @deini and @johannestroeger ## 0.5.0 * Dropped support for Elixir 1.4 and OTP 18 releases. You will need to use Elixir 1.5+ and OTP 19+. * Switched to running our tests against gnatsd `1.3.0` ================================================ FILE: LICENSE.txt ================================================ Copyright 2017 Michael Ries Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. ================================================ FILE: MAINTAINERS.md ================================================ # Maintainers Maintainership is on a per project basis. ### Maintainers - Colin Sullivan [@ColinSullivan1](https://github.com/ColinSullivan1) - Michael Ries [@mmmries](https://github.com/mmmries) - Kevin Hoffman [@autodidaddict](https//github.com/autodidaddict) ================================================ FILE: README.md ================================================ [![hex.pm](https://img.shields.io/hexpm/v/gnat.svg)](https://hex.pm/packages/gnat) [![hex.pm](https://img.shields.io/hexpm/dt/gnat.svg)](https://hex.pm/packages/gnat) [![hex.pm](https://img.shields.io/hexpm/l/gnat.svg)](https://hex.pm/packages/gnat) [![github.com](https://img.shields.io/github/last-commit/nats-io/nats.ex.svg)](https://github.com/nats-io/nats.ex) ![NATS](https://nats.io/img/logos/nats-horizontal-color.png) # Gnat A [nats.io](https://nats.io/) client for Elixir. The goals of the project are resiliency, performance, and ease of use. > Hex documentation available here: https://hex.pm/packages/gnat ## Usage ``` elixir {:ok, gnat} = Gnat.start_link(%{host: "127.0.0.1", port: 4222}) # Or if the server requires TLS you can start a connection with: # {:ok, gnat} = Gnat.start_link(%{host: "127.0.0.1", port: 4222, tls: true}) {:ok, subscription} = Gnat.sub(gnat, self(), "pawnee.*") :ok = Gnat.pub(gnat, "pawnee.news", "Leslie Knope recalled from city council (Jammed)") receive do {:msg, %{body: body, topic: "pawnee.news", reply_to: nil}} -> IO.puts(body) end ``` ## Authentication ``` elixir # with user and password {:ok, gnat} = Gnat.start_link(%{host: "127.0.0.1", port: 4222, username: "joe", password: "123", auth_required: true}) # with token {:ok, gnat} = Gnat.start_link(%{host: "127.0.0.1", port: 4222, token: "secret", auth_required: true}) # with an nkey seed {:ok, gnat} = Gnat.start_link(%{host: "127.0.0.1", port: 4222, nkey_seed: "SUAM...", auth_required: true}) # with decentralized user credentials (JWT) {:ok, gnat} = Gnat.start_link(%{host: "127.0.0.1", port: 4222, nkey_seed: "SUAM...", jwt: "eyJ0eX...", auth_required: true}) # connect to NGS with JWT {:ok, gnat} = Gnat.start_link(%{host: "connect.ngs.global", tls: true, jwt: "ey...", nkey_seed: "SUAM..."}) ``` ## TLS Connections [NATS Server](https://github.com/nats-io/nats-server) is often configured to accept or require TLS connections. In order to connect to these clusters you'll want to pass some extra TLS settings to your `Gnat` connection. ``` elixir # using a basic TLS connection {:ok, gnat} = Gnat.start_link(%{host: "127.0.0.1", port: 4222, tls: true}) # Passing a Client Certificate for verification {:ok, gnat} = Gnat.start_link(%{tls: true, ssl_opts: [certfile: "client-cert.pem", keyfile: "client-key.pem"]}) ``` ## Resiliency If you would like to stay connected to a cluster of nats servers, you should consider using `Gnat.ConnectionSupervisor` . This can be added to your supervision tree in your project and will handle automatically re-connecting to the cluster. For long-lived subscriptions consider using `Gnat.ConsumerSupervisor` . This can also be added to your supervision tree and use a supervised connection to re-establish a subscription. It also handles details like handling each message in a supervised process so you isolate failures and get OTP logs when an unexpected error occurs. ## Services If you supply a module that implements the `Gnat.Services.Server` behavior and the `service_definition` configuration field to a `Gnat.ConsumerSupervisor`, then this client will automatically take care of exposing the service to discovery, responding to pings, and maintaining and exposing statistics like request and error counts, and processing times. ## Instrumentation Gnat uses [telemetry](https://hex.pm/packages/telemetry) to make instrumentation data available to clients. If you want to record metrics around the number of messages or latency of message publishes, subscribes, requests, etc you can do the following in your project: ``` elixir iex(1)> metrics_function = fn(event_name, measurements, event_meta, config) -> IO.inspect([event_name, measurements, event_meta, config]) :ok end #Function<4.128620087/4 in :erl_eval.expr/5> iex(2)> names = [[:gnat, :pub], [:gnat, :sub], [:gnat, :message_received], [:gnat, :request], [:gnat, :unsub]] [ [:gnat, :pub], [:gnat, :sub], [:gnat, :message_received], [:gnat, :request], [:gnat, :unsub], [:gnat, :service_request], [:gnat, :service_error] ] iex(3)> :telemetry.attach_many("my listener", names, metrics_function, %{my_config: true}) :ok iex(4)> {:ok, gnat} = Gnat.start_link() {:ok, #PID<0.203.0>} iex(5)> Gnat.sub(gnat, self(), "topic") [[:gnat, :sub], %{latency: 128000}, %{topic: "topic"}, %{my_config: true}] {:ok, 1} iex(6)> Gnat.pub(gnat, "topic", "ohai") [[:gnat, :pub], %{latency: 117000}, %{topic: "topic"}, %{my_config: true}] [[:gnat, :message_received], %{count: 1}, %{topic: "topic"}, %{my_config: true}] :ok ``` The `pub` , `sub` , `request` , and `unsub` events all report the latency of those respective calls. The `message_received` event reports a number of messages like `%{count: 1}` because there isn't a good latency metric to report. Any microservices managed by a consumer supervisor will also report `service_request` and `service_error`. In addition to the `:topic` metadata, microservices will also include `:endpoint` and `:group` (which can be `nil`) in their telemetry reports. All of the events (except `unsub` ) include metadata with a `:topic` key so you can split your metrics by topic. ## Benchmarks Part of the motivation for building this library is to get better performance. To this end, there is a `bench` branch on this project which includes a `server.exs` and `client.exs` that can be used for benchmarking various scenarios. As of this commit, the [latest benchmark on a 16-core server](https://gist.github.com/mmmries/08fe44fdd47a6f8838936f41170f270a) shows that you can make 170k+ req/sec or up to 192MB/sec. The `bench/*.exs` files also contain some straight-line single-CPU performance tests. As of this commit my 2018 MacBook pro shows. | | ips | average | deviation | median | | ------------- | -------- | --------- | --------- | ------ | | parse-128 | 487.67 K | 2.19 μs | ±1701.54% | 2 μs | | pub - 128 | 96.37 K | 10.38 μs | ±102.94% | 10 μs | | req-reply-128 | 8.32 K | 120.16 μs | ±23.68% | 114 μs | ## Development Before running the tests make sure you have a locally running copy of `nats-server` ([installation instructions](https://docs.nats.io/running-a-nats-service/introduction/installation)). By default, tests are run with no authentication. Make sure your NATS configuration contains no users, or has an account with [no_auth_user](https://docs.nats.io/running-a-nats-service/configuration/securing_nats/accounts#no-auth-user) explicitly enabled. We currently use version `2.10.24` in CI, but anything higher than `2.2.0` should be fine. Versions from `0.9.6` up to `2.2.0` should work fine for everything except header support. Make sure to enable jetstream with the `nats-server -js` argument and you might also want to enable debug and verbose logging if you're trying to understand the messages being sent to/from nats (ie `nats-server -js -D -V`). The typical `mix test` will run all the basic unit tests. You can also run the `multi_server` set of tests that test connectivity to different `nats-server` configurations. You can run these with `mix test --only multi_server` . The tests will tell you how to start the different configurations. There are also some property-based tests that generate a lot of test cases. You can tune how many test cases by setting the environment variable `N=200 mix test --only property` (default it 100). For more details you can look at how Github runs these things in the CI flow. ================================================ FILE: bench/client.exs ================================================ Application.put_env(:client, :num_connections, 4) num_requesters = 16 requests_per_requester = 500 defmodule Client do require Logger def setup(id) do num_connections = Application.get_env(:client, :num_connections) partition = rem(id, num_connections) String.to_atom("gnat#{partition}") end def send_request(gnat, request) do {:ok, _} = Gnat.request(gnat, "echo", request)# |> IO.inspect end def send_requests(gnat, how_many, request) do :lists.seq(1, how_many) |> Enum.each(fn(_) -> {micro_seconds, _result} = :timer.tc(fn() -> send_request(gnat, request) end) Benchmark.record_rpc_time(micro_seconds) end) end end defmodule Benchmark do def benchmark(num_actors, requests_per_actor, request) do {:ok, _pid} = Agent.start_link(fn -> [] end, name: __MODULE__) {total_micros, _result} = time_benchmark(num_actors, requests_per_actor, request) total_requests = num_actors * requests_per_actor total_bytes = total_requests * byte_size(request) * 2 print_statistics(total_requests, total_bytes, total_micros) Agent.stop(__MODULE__, :normal) end def record_rpc_time(micro_seconds) do Agent.update(__MODULE__, fn(list) -> [micro_seconds | list] end) end def print_statistics(total_requests, total_bytes, total_micros) do total_seconds = total_micros / 1_000_000.0 req_throughput = total_requests / total_seconds kilobyte_throughput = total_bytes / 1024 / total_seconds IO.puts "It took #{total_seconds}sec" IO.puts "\t#{req_throughput} req/sec" IO.puts "\t#{kilobyte_throughput} kb/sec" Agent.get(__MODULE__, fn(list_of_rpc_times) -> tc_l = list_of_rpc_times tc_n = Enum.count(list_of_rpc_times) tc_min = :lists.min(tc_l) tc_max = :lists.max(tc_l) sorted = :lists.sort(tc_l) tc_med = :lists.nth(round(tc_n * 0.5), sorted) tc_90th = :lists.nth(round(tc_n * 0.9), sorted) tc_avg = round(Enum.sum(tc_l) / tc_n) IO.puts "\tmin: #{tc_min}µs" IO.puts "\tmax: #{tc_max}µs" IO.puts "\tmedian: #{tc_med}µs" IO.puts "\t90th percentile: #{tc_90th}µs" IO.puts "\taverage: #{tc_avg}µs" IO.puts "\t#{tc_min},#{tc_max},#{tc_med},#{tc_90th},#{tc_avg},#{req_throughput},#{kilobyte_throughput}" end) end def time_benchmark(num_actors, requests_per_actor, request) do :timer.tc(fn() -> (1..num_actors) |> Enum.map(fn(i) -> parent = self() spawn(fn() -> gnat = Client.setup(i) #IO.puts "starting requests #{i}" Client.send_requests(gnat, requests_per_actor, request) #IO.puts "done with requests #{i}" send parent, :ack end) end) wait_for_times(num_actors) end) end def wait_for_times(0), do: :done def wait_for_times(n) do receive do :ack -> wait_for_times(n-1) end end end num_connections = Application.get_env(:client, :num_connections) Enum.each(0..(num_connections - 1), fn(i) -> name = :"gnat#{i}" {:ok, _pid} = Gnat.start_link(%{}, name: name) end) :timer.sleep(500) # let the connections get started #request = "ping" request = :crypto.strong_rand_bytes(16) Benchmark.benchmark(num_requesters, requests_per_requester, request) ================================================ FILE: bench/kv_consume.exs ================================================ # bench/kv_consume.exs # # Compares three approaches for consuming all messages from a KV bucket into ETS: # # 1. Pager (batch 500, ack_policy: :all) — fetch a page, process, ack last, repeat # 2. Pull + ack_next pipeline (initial batch 500, ack_policy: :explicit) — prime the # pipeline with a batch request, then ack_next each message to keep flow continuous # 3. PullConsumer with batch_size (ack_policy: :all) — the new batch mode using the # actual PullConsumer behaviour, batches messages and acks only the last per batch # # Prerequisites: # - NATS server with JetStream enabled: nats-server -js # - Run with: mix run bench/kv_consume.exs # # Optional env vars: # - BENCH_COUNT: number of messages (default 100000) # - BENCH_BATCH: batch size (default 500) # - BENCH_TIME: seconds per scenario (default 60) Logger.configure(level: :warning) require Logger Logger.configure(level: :warning) alias Gnat.Jetstream.API.{Consumer, KV} alias Gnat.Jetstream.API.Util defmodule BenchBatchPullConsumer do use Gnat.Jetstream.PullConsumer def start(args) do Gnat.Jetstream.PullConsumer.start(__MODULE__, args) end @impl true def init(%{tab: tab, notify: pid, expected: expected, batch_size: batch_size}) do consumer = %Consumer{ stream_name: "KV_BENCH_KV", ack_policy: :all, ack_wait: 30_000_000_000, deliver_policy: :all, replay_policy: :instant } {:ok, %{tab: tab, notify: pid, expected: expected, received: 0}, connection_name: :gnat_bench, consumer: consumer, batch_size: batch_size} end @impl true def handle_message(message, state) do :ets.insert(state.tab, {message.topic, message.body}) received = state.received + 1 if received >= state.expected do send(state.notify, {:done, received}) end {:ack, %{state | received: received}} end end defmodule KVConsumeBench do @bucket "BENCH_KV" @stream "KV_BENCH_KV" @value_size 64 def setup(conn, count) do # Clean up previous state KV.delete_bucket(conn, @bucket) :timer.sleep(500) {:ok, _} = KV.create_bucket(conn, @bucket, history: 1) IO.puts("Populating #{count} messages (#{@value_size} byte values)...") start = System.monotonic_time(:millisecond) Enum.each(1..count, fn i -> key = "key.#{String.pad_leading(Integer.to_string(i), 7, "0")}" value = :crypto.strong_rand_bytes(@value_size) |> Base.encode64() :ok = KV.put_value(conn, @bucket, key, value) if rem(i, 10_000) == 0 do elapsed = System.monotonic_time(:millisecond) - start rate = round(i / elapsed * 1000) IO.puts(" #{i}/#{count} (#{rate} msg/s)") end end) elapsed = System.monotonic_time(:millisecond) - start IO.puts("Setup complete: #{count} messages in #{div(elapsed, 1000)}s\n") end # --------------------------------------------------------------------------- # Strategy 1: Pager (ack_policy: :all, batch fetch, ack last per page) # --------------------------------------------------------------------------- def pager_consume(conn, batch_size) do tab = :ets.new(:pager_cache, [:set]) {:ok, _} = Gnat.Jetstream.Pager.reduce(conn, @stream, [batch: batch_size], nil, fn msg, acc -> :ets.insert(tab, {msg.topic, msg.body}) acc end) count = :ets.info(tab, :size) :ets.delete(tab) count end # --------------------------------------------------------------------------- # Strategy 2: Pull + ack_next pipeline (ack_policy: :explicit, continuous) # --------------------------------------------------------------------------- def pull_ack_next_consume(conn, batch_size) do {:ok, consumer_info} = Consumer.create(conn, %Consumer{ stream_name: @stream, ack_policy: :explicit, deliver_policy: :all, replay_policy: :instant, inactive_threshold: 30_000_000_000 }) total = consumer_info.num_pending inbox = Util.reply_inbox() {:ok, sub} = Gnat.sub(conn, self(), inbox) :ok = Consumer.request_next_message( conn, @stream, consumer_info.name, inbox, nil, batch: batch_size, no_wait: true ) tab = :ets.new(:pull_cache, [:set]) receive_with_ack_next(sub, inbox, tab, 0, total) count = :ets.info(tab, :size) :ets.delete(tab) Gnat.unsub(conn, sub) Consumer.delete(conn, @stream, consumer_info.name) count end @terminals ["404", "408"] defp receive_with_ack_next(_sub, _inbox, _tab, total, total), do: :ok defp receive_with_ack_next(sub, inbox, tab, count, total) do receive do {:msg, %{sid: ^sub, status: status}} when status in @terminals -> receive_with_ack_next(sub, inbox, tab, count, total) {:msg, %{sid: ^sub, reply_to: nil}} -> receive_with_ack_next(sub, inbox, tab, count, total) {:msg, %{sid: ^sub} = message} -> :ets.insert(tab, {message.topic, message.body}) if count + 1 < total do Gnat.Jetstream.ack_next(message, inbox) else Gnat.Jetstream.ack(message) end receive_with_ack_next(sub, inbox, tab, count + 1, total) after 30_000 -> IO.puts("WARNING: timeout after receiving #{count}/#{total} messages") :timeout end end # --------------------------------------------------------------------------- # Strategy 3: PullConsumer with batch_size (ack_policy: :all, batch mode) # # Uses the actual PullConsumer behaviour with the new batch_size option. # This is the real-world usage pattern we want to validate. # --------------------------------------------------------------------------- def batch_pull_consumer_consume(expected, batch_size) do tab = :ets.new(:batch_pc_cache, [:set, :public]) {:ok, pid} = BenchBatchPullConsumer.start(%{ tab: tab, notify: self(), expected: expected, batch_size: batch_size }) receive do {:done, _received} -> :ok after 60_000 -> IO.puts("WARNING: PullConsumer timeout") end count = :ets.info(tab, :size) Gnat.Jetstream.PullConsumer.close(pid) :ets.delete(tab) count end end # -- Configuration ----------------------------------------------------------- count = String.to_integer(System.get_env("BENCH_COUNT", "100000")) batch = String.to_integer(System.get_env("BENCH_BATCH", "500")) time = String.to_integer(System.get_env("BENCH_TIME", "60")) IO.puts(""" KV Consume Benchmark ==================== Messages: #{count} Batch size: #{batch} Time/scenario: #{time}s """) # -- Setup -------------------------------------------------------------------- # Named connection for the PullConsumer conn_settings = %{ name: :gnat_bench, backoff_period: 1_000, connection_settings: [%{host: '127.0.0.1', port: 4222}] } {:ok, _} = Gnat.ConnectionSupervisor.start_link(conn_settings) :timer.sleep(500) # Direct connection for Pager and manual pull {:ok, conn} = Gnat.start_link(%{host: '127.0.0.1', port: 4222}) KVConsumeBench.setup(conn, count) # -- Verify all approaches produce correct results ---------------------------- IO.puts("Verifying approaches...") count1 = KVConsumeBench.pager_consume(conn, batch) IO.puts(" Pager: #{count1} entries") count2 = KVConsumeBench.pull_ack_next_consume(conn, batch) IO.puts(" Pull+ack_next: #{count2} entries") count3 = KVConsumeBench.batch_pull_consumer_consume(count, batch) IO.puts(" Batch PullConsumer: #{count3} entries") expected_counts = [count1, count2, count3] if Enum.any?(expected_counts, &(&1 != count)) do IO.puts("\nERROR: expected #{count} entries from each approach") Gnat.stop(conn) System.halt(1) end IO.puts("\nAll approaches verified. Starting benchmark...\n") # -- Benchmark --------------------------------------------------------------- Benchee.run( %{ "pager (batch #{batch})" => fn -> KVConsumeBench.pager_consume(conn, batch) end, "pull+ack_next (initial batch #{batch})" => fn -> KVConsumeBench.pull_ack_next_consume(conn, batch) end, "batch_pull_consumer (batch #{batch})" => fn -> KVConsumeBench.batch_pull_consumer_consume(count, batch) end }, time: time, warmup: 0, memory_time: 0, formatters: [{Benchee.Formatters.Console, comparisons: true}] ) Gnat.stop(conn) ================================================ FILE: bench/parse.exs ================================================ msg1024 = :crypto.strong_rand_bytes(1024) msg128 = :crypto.strong_rand_bytes(128) msg16 = :crypto.strong_rand_bytes(16) inputs = %{ "16 byte" => "MSG topic 1 16\r\n#{msg16}\r\n", "128 byte" => "MSG topic 1 128\r\n#{msg128}\r\n", "1024 byte" => "MSG topic 1 1024\r\n#{msg1024}\r\n", "7 byte with headers" => "HMSG SUBJECT 1 REPLY 48 55\r\nNATS/1.0\r\nHeader1: X\r\nHeader1: Y\r\nHeader2: Z\r\n\r\nPAYLOAD\r\n" } parsec = Gnat.Parsec.new() Benchee.run(%{ "parsec" => fn(tcp_packet) -> {_parse, [_msg]} = Gnat.Parsec.parse(parsec, tcp_packet) end, }, time: 10, parallel: 1, inputs: inputs, formatters: [{Benchee.Formatters.Console, comparisons: false}]) ================================================ FILE: bench/publish.exs ================================================ inputs = %{ "16 byte" => :crypto.strong_rand_bytes(16), "128 byte" => :crypto.strong_rand_bytes(128), "1024 byte" => :crypto.strong_rand_bytes(1024), } {:ok, client_pid} = Gnat.start_link(%{host: '127.0.0.1', port: 4222}) Benchee.run(%{ "pub" => fn(msg) -> :ok = Gnat.pub(client_pid, "echo", msg) end, }, time: 10, parallel: 1, inputs: inputs, formatters: [{Benchee.Formatters.Console, comparisons: false}]) ================================================ FILE: bench/request.exs ================================================ defmodule EchoServer do def run(gnat) do spawn(fn -> init(gnat) end) end def init(gnat) do Gnat.sub(gnat, self(), "echo") loop(gnat) end def loop(gnat) do receive do {:msg, %{topic: "echo", reply_to: reply_to, body: msg}} -> Gnat.pub(gnat, reply_to, msg) other -> IO.puts "server received: #{inspect other}" end loop(gnat) end end {:ok, server_pid} = Gnat.start_link(%{host: '127.0.0.1', port: 4222}) EchoServer.run(server_pid) inputs = %{ "16 byte" => :crypto.strong_rand_bytes(16), "128 byte" => :crypto.strong_rand_bytes(128), "1024 byte" => :crypto.strong_rand_bytes(1024), } {:ok, client_pid} = Gnat.start_link(%{host: '127.0.0.1', port: 4222}) Benchee.run(%{ "request" => fn(msg) -> {:ok, %{body: _}} = Gnat.request(client_pid, "echo", msg) end, }, time: 10, parallel: 1, inputs: inputs, formatters: [{Benchee.Formatters.Console, comparisons: false}]) ================================================ FILE: bench/request_multi.exs ================================================ defmodule EchoServer do def run(gnat) do spawn(fn -> init(gnat) end) end def init(gnat) do Gnat.sub(gnat, self(), "echo") loop(gnat) end def loop(gnat) do receive do {:msg, %{topic: "echo", reply_to: reply_to, body: msg}} -> Gnat.pub(gnat, reply_to, msg) other -> IO.puts "server received: #{inspect other}" end loop(gnat) end end {:ok, server_pid} = Gnat.start_link(%{host: '127.0.0.1', port: 4222}) # run 3 servers to get 3 responses EchoServer.run(server_pid) EchoServer.run(server_pid) EchoServer.run(server_pid) inputs = %{ "16 byte" => :crypto.strong_rand_bytes(16), "128 byte" => :crypto.strong_rand_bytes(128), "1024 byte" => :crypto.strong_rand_bytes(1024), } {:ok, client_pid} = Gnat.start_link(%{host: '127.0.0.1', port: 4222}) Benchee.run(%{ "request_multi" => fn(msg) -> {:ok, [%{body: _}, %{}, %{}]} = Gnat.request_multi(client_pid, "echo", msg, max_messages: 3) end, }, time: 10, parallel: 1, inputs: inputs, formatters: [{Benchee.Formatters.Console, comparisons: false}]) ================================================ FILE: bench/server.exs ================================================ num_connections = 4 num_subscribers = 4 Enum.each(0..(num_connections - 1), fn(i) -> name = :"gnat#{i}" {:ok, _pid} = Gnat.start_link(%{}, name: name) end) Enum.each(0..(num_subscribers - 1), fn(i) -> name = :"consumer#{i}" conn_name = :"gnat#{rem(i, num_connections)}" IO.puts "#{name} will use #{conn_name}" {:ok, _pid} = Gnat.ConsumerSupervisor.start_link(%{connection_name: conn_name, consuming_function: {EchoServer, :handle}, subscription_topics: [%{topic: "echo", queue_group: "echo"}]}) end) defmodule EchoServer do def handle(%{body: body, reply_to: reply_to, gnat: gnat_pid}) do Gnat.pub(gnat_pid, reply_to, body) end def wait_loop do :timer.sleep(1_000) wait_loop() end end EchoServer.wait_loop() ================================================ FILE: bench/service_bench.exs ================================================ defmodule EchoService do use Gnat.Services.Server def request(%{body: body}, "echo", _group) do {:reply, body} end def definition do %{ name: "echo", description: "This is an example service", version: "0.0.1", endpoints: [ %{ name: "echo", group_name: "mygroup", } ] } end end conn_supervisor_settings = %{ name: :gnat, # (required) the registered named you want to give the Gnat connection backoff_period: 1_000, # number of milliseconds to wait between consecutive reconnect attempts (default: 2_000) connection_settings: [ %{host: '127.0.0.1', port: 4222}, ] } {:ok, _pid} = Gnat.ConnectionSupervisor.start_link(conn_supervisor_settings) # let the connection get established :timer.sleep(100) consumer_supervisor_settings = %{ connection_name: :gnat, module: EchoService, # a module that implements the Gnat.Services.Server behaviour service_definition: EchoService.definition() } {:ok, _pid} = Gnat.ConsumerSupervisor.start_link(consumer_supervisor_settings) # wait for the connection and consumer to be ready :timer.sleep(2000) {:ok, client_pid} = Gnat.start_link(%{host: '127.0.0.1', port: 4222}) inputs = %{ "16 byte" => :crypto.strong_rand_bytes(16), "256 byte" => :crypto.strong_rand_bytes(256), "1024 byte" => :crypto.strong_rand_bytes(1024), } Benchee.run(%{ "service" => fn(msg) -> {:ok, %{body: ^msg}} = Gnat.request(client_pid, "mygroup.echo", msg) end, }, time: 10, parallel: 1, inputs: inputs, formatters: [{Benchee.Formatters.Console, comparisons: false}]) ================================================ FILE: dependencies.md ================================================ # Project Dependencies This is a list of dependencies that will be pulled into your project when you use this library. Unless otherwise specified all dependencies are Hex packages obtained via mix. This list of dependencies was produced on October 13 2023. | Dependency | License | |-|-| | ed25519 1.4.0 | MIT | | cowlib 2.11.0 | ISC | | deep_merge 1.0.0 | MIT | | jason 1.2.2 | Apache 2.0 | | nimble_parsec 1.2.0 | Apache 2.0 | | nkeys 0.2.1 | MIT | | telemetry 1.0.0 (rebar3) | Apache 2.0 | # Development Dependencies This is a list of dependencies used to build and test this library. | Dependency | License | |-|-| | propcheck 1.4.1 | GPL 3.0 | | proper 1.4.0 | GPL 3.0 | | erlex 0.2.6 | Apache 2.0 | | makeup 1.0.5 | BSD | | makeup_elixir 0.15.2 | BSD | | makeup_erlang 0.1.1 | BSD | | deep_merge 0.2.0 | MIT | | benchee 1.0.1 | MIT | | dialyxir 1.1.0 | Apache 2.0 | | earmark 1.3.1 | Apache 2.0 | | makeup_elixir 0.13.0 | BSD | | earmark_parser 1.4.18 | Apache-2.0 | | ex_doc 0.26.0 | Apache 2.0 | ================================================ FILE: docs/js/guides/broadway.md ================================================ # Using Broadway with Jetstream Broadway is a library which allows building concurrent and multi-stage data ingestion and data processing pipelines with Elixir easily. You can learn about it more in [Broadway documentation](https://hexdocs.pm/broadway/introduction.html). Jetstream library comes with tools necessary to use NATS Jetstream with Broadway. ## Getting started In order to use Broadway with NATS Jetstream you need to: 1. Setup a NATS Server with JetStream turned on 2. Create stream and consumer on NATS server 3. Configure Gnat connection in your Elixir project 4. Configure your project to use Broadway In this guide, we are going to focus on the fourth point. To learn how to start Jetstream locally with Docker Compose and then add Gnat and Jetstream to your application, see the Starting Jetstream section in [Getting Started guide](../introduction/getting_started.md). ### Adding Broadway to your application Once we have NATS with JetStream running and the stream and consumer we are going to use are created, we can proceed to adding Broadway to our project. First, put `:broadway` to the list of dependencies in `mix.exs`. ```elixir defp deps do [ ... {:broadway, ...version...}, ... ] end ``` Visit [Broadway page on Hex.pm](https://hex.pm/packages/broadway) to check for current version to put in `deps`. To install the dependencies, run: ```shell mix deps.get ``` ### Defining the pipeline configuration The next step is to define your Broadway module. We need to implement three functions in order to define a Broadway pipeline: `start_link/1`, `handle_message/3` and `handle_batch/4`. Let's create `start_link/1` first: ```elixir defmodule MyBroadway do use Broadway alias Broadway.Message def start_link(_opts) do Broadway.start_link( __MODULE__, name: MyBroadwayExample, producer: [ module: { OffBroadway.Jetstream.Producer, connection_name: :gnat, stream_name: "TEST_STREAM", consumer_name: "TEST_CONSUMER" }, concurrency: 10 ], processors: [ default: [concurrency: 10] ], batchers: [ default: [ concurrency: 5, batch_size: 10, batch_timeout: 2_000 ] ] ... ) end ...callbacks.. end ``` All `start_link/1` does is just delegating to `Broadway.start_link/2`. To understand what all these options mean and to learn about other possible settings, visit [Broadway documentation](https://hexdocs.pm/broadway/Broadway.html). The part that interests us the most in this guide is the `producer.module`. Here we're choosing `OffBroadway.Jetstream.Producer` as the producer module and passing the connection options, such as Gnat process name and stream name. For full list of available options, visit [Producer](`OffBroadway.Jetstream.Producer`) documentation. ### Implementing Broadway callbacks Broadway requires some callbacks to be implemented in order to process messages. For full list of available callbacks visit [Broadway documentation](https://hexdocs.pm/broadway/Broadway.html#callbacks). A simple example: ```elixir defmodule MyBroadway do use Broadway alias Broadway.Message ...start_link... def handle_message(_processor_name, message, _context) do message |> Message.update_data(&process_data/1) |> case do "FOO" -> Message.configure_ack(on_success: :term) "BAR" -> Message.configure_ack(on_success: :nack) message -> message end end defp process_data(data) do String.upcase(data) end def handle_batch(_, messages, _, _) do list = messages |> Enum.map(fn e -> e.data end) IO.puts("Got a batch: #{inspect(list)}. Sending acknowledgements...") messages end ``` First, in `handle_message/3` we update our messages' data individually by converting them to uppercase. Then, in the same callback, we're changing the success ack option of the message to `:term` if its content is `"FOO"` or to `:nack` if the message is `"BAR"`. In the end we print each batch in `handle_batch/4`. It's not quite useful but should be enough for this guide. ## Running the Broadway pipeline Once we have our pipeline fully defined, we need to add it as a child in the supervision tree. Most applications have a supervision tree defined at `lib/my_app/application.ex`. ```elixir children = [ {MyBroadway, []} ] Supervisor.start_link(children, strategy: :one_for_one) ``` You can now test the pipeline. Let's start the application: ```shell iex -S mix ``` Use Gnat API to send messages to your stream: ```elixir Gnat.pub(:gnat, "test_subject", "foo") Gnat.pub(:gnat, "test_subject", "bar") Gnat.pub(:gnat, "test_subject", "baz") ``` Batcher should then print: ``` Got a batch: ["FOO", "BAR", "BAZ"]. Sending acknowledgements... ``` ================================================ FILE: docs/js/guides/managing.md ================================================ # Managing Streams and Consumers Jetstream provides a JSON API for managing streams and consumers. This library exposes this API via interactions with the `Jetstream.Api.Stream` and `Jetstream.Api.Consumer` modules. These modules act as native wrappers for the API and do not attempt to simplify any of the common use-cases. As this library matures we may introduce a separate layer of functions to handle these scenarios, but for now our aim is to provide full access to the Jetstream API. ================================================ FILE: docs/js/guides/push_based_consumer.md ================================================ # Push based consumer ```elixir # Start a nats server with jetstream enabled and default configs # Now run the following snippets in an IEx terminal alias Jetstream.API.{Consumer,Stream} # Setup a connection to the nats server and create the stream/consumer # This is the equivalent of these two nats cli commands # nats stream add TEST --subjects="greetings" --max-msgs=-1 --max-msg-size=-1 --max-bytes=-1 --max-age=-1 --storage=file --retention=limits --discard=old # nats consumer add TEST TEST --target consumer.greetings --replay instant --deliver=all --ack all --wait=5s --filter="" --max-deliver=10 {:ok, connection} = Gnat.start_link() stream = %Stream{name: "TEST", subjects: ["greetings"]} {:ok, _response} = Stream.create(connection, stream) consumer = %Consumer{stream_name: "TEST", name: "TEST", deliver_subject: "consumer.greetings", ack_wait: 5_000_000_000, max_deliver: 10} {:ok, _response} = Consumer.create(connection, consumer) # Setup Consuming Function defmodule Subscriber do def handle(msg) do IO.inspect(msg) case msg.body do "hola" -> Jetstream.ack(msg) "bom dia" -> Jetstream.nack(msg) _ -> nil end end end # normally you would add the `ConnectionSupervisor` and `ConsumerSupervisor` to your supervisrion tree # here we start them up manually in an IEx session {:ok, _pid} = Gnat.ConnectionSupervisor.start_link(%{ name: :gnat, backoff_period: 4_000, connection_settings: [ %{} ] }) {:ok, _pid} = Gnat.ConsumerSupervisor.start_link(%{ connection_name: :gnat, consuming_function: {Subscriber, :handle}, subscription_topics: [ %{topic: "consumer.greetings"} ] }) # now publish some messages into the stream Gnat.pub(:gnat, "greetings", "hello") # no ack will be sent back, so you'll see this message received 10 times with a 5sec pause between each one Gnat.pub(:gnat, "greetings", "hola") # an ack is sent back so this will only be received once Gnat.pub(:gnat, "greetings", "bom dia") # a -NAK is sent back so you'll see this received 10 times very quickly ``` ================================================ FILE: docs/js/introduction/getting_started.md ================================================ # Getting Started In this guide, we're going to learn how to install Jetstream in your project and start consuming messages from your streams. ## Starting Jetstream The following Docker Compose file will do the job: ```yaml version: "3" services: nats: image: nats:latest command: - -js ports: - 4222:4222 ``` Save this snippet as `docker-compose.yml` and run the following command: ```shell docker compose up -d ``` Let's also create Jetstream stream where we will publish our hello world messages: ```shell nats stream add HELLO --subjects="greetings" ``` > #### Tip {: .tip} > > You can also manage Jetstream streams and consumers via Elixir. You can see more details in > [this guide](../guides/managing.md). ## Adding Jetstream and Gnat to an application To start off with, we'll generate a new Elixir application by running this command: ``` mix new hello_jetstream --sup ``` We need to have [a supervision tree](http://elixir-lang.org/getting-started/mix-otp/supervisor-and-application.html) up and running in your app, and the `--sup` option ensures that. To add Jetstream to this application, you need to add [Jetstream](https://hex.pm/packages/jetstream) and [Gnat](https://hex.pm/packages/gnat) libraries to your `deps` definition in our `mix.exs` file. **Fill exact version requirements from each package Hex.pm pages.** ```elixir defp deps do [ {:gnat, ...}, {:jetstream, ...} ] end ``` To install these dependencies, we will run this command: ```shell mix deps.get ``` Now let's connect to our NATS server. To do this, you need to start `Gnat.ConnectionSupervisor` under our application's supervision tree. Add following to `lib/hello_jetstream/application.ex`: ```elixir def start(_type, _args) do children = [ ... # Create NATS connection {Gnat.ConnectionSupervisor, %{ name: :gnat, connection_settings: [ %{host: "localhost", port: 4222} ] }}, ] ... ``` This piece of configuration will start Gnat processes that connect to the NATS server and allow publishing and subscribing to any subjects. Jetstream operates using plain NATS subjects which follow specific naming and message format conventions. Let's now create a _pull consumer_ which will subscribe a specific Jetstream stream and print incoming messages to standard output. ## Creating a pull consumer Jetstream requires us to allocate a view/cursor of the stream that our consumer will operate on. In Jetstream terminology, this view is called a _consumer_ (Funnily enough we've just implemented a consumer in our code, coincidence?). [Jetstream](https://docs.nats.io/nats-concepts/jetstream/consumers) [documentation](https://docs.nats.io/nats-concepts/jetstream/consumers/example_configuration) offers great insights on benefits of having this separate concept so we won't duplicate work here. Jetstream offers two stream consuming modes: _push_ and _pull_. In _push_ mode, Jetstream will simply send messages to selected consumers immediately when they are received. This approach does offer congestion control, so it is not recommended for high-volume and/or reliability sensitive streams. You do not really need this library to implement push consumer because all building blocks are in `Gnat` library. You can read more about push consumers in [this guide](../guides/push_based_consumer.md). On the other hand, in _pull_ mode consumers ask Jetstream for more messages when they are ready to process them. This is the recommended approach for most use cases and we will proceed with it in this guide. > #### This is just a brief outline {: .tip} > > For more details about differences between consumer modes, consult > [Jetstream documentation](https://docs.nats.io/nats-concepts/jetstream/consumers). Let's create a pull consumer module within our application at `lib/hello_jetstream/logger_pull_consumer.ex`: ```elixir defmodule HelloJetstream.LoggerPullConsumer do use Jetstream.PullConsumer def start_link([]) do Jetstream.PullConsumer.start_link(__MODULE__, []) end @impl true def init([]) do {:ok, nil, connection_name: :gnat, stream_name: "HELLO", consumer_name: "LOGGER"} end @impl true def handle_message(message, state) do IO.inspect(message) {:ack, state} end end ``` Pull Consumer is a regular `GenServer` and it takes a reference to `Gnat.ConnectionSupervisor` along with names of Jetstream stream and consumer as options passed to `Jetstream.PullConsumer.start*` functions. These options are passed as keyword list in third element of tuple returned from the `c:Jetstream.PullConsumer.init/1` callback. The only required callbacks are well known gen server's `c:Jetstream.PullConsumer.init/1` and `c:Jetstream.PullConsumer.handle_message/2`, which takes new message as its first argument and is expected to return an _ACK action_ instructing underlying process loop what to do with this message. Here we are asking it to automatically send for us an ACK message back to Jetstream. Let's now create a consumer in our NATS server. We will call it `LOGGER` as we plan to let it simply log everything published to the stream. ```shell nats consumer add --pull --deliver=all HELLO LOGGER ``` Now, let's start our pull consumer under application's supervision tree. ```elixir def start(_type, _args) do children = [ ... # Jetstream Pull Consumer HelloJetstream.LoggerPullConsumer, ] ... ``` Let's now publish some messages to our `HELLO` stream, so something will be waiting for our application to be read when it starts. ## Publishing messages to streams Jetstream listens on regular NATS subjects, so publishing messages is dead simple with `Gnat.pub/3`: ```elixir Gnat.pub(:gnat, "greetings", "Hello World") ``` Or via NATS CLI: ```shell nats pub greetings "Hello World" ``` That's it! When you run your app, you should see your messages being read by your application. ================================================ FILE: docs/js/introduction/overview.md ================================================ # Overview [Jetstream](https://docs.nats.io/nats-concepts/jetstream) is a distributed persistence system built-in to [NATS](https://nats.io/). It provides a streaming system that lets you capture streams of events from various sources and persist these into persistent stores, which you can immediately or later replay for processing. This library exposes interfaces for publishing, consuming and managing Jetstream services. It builds on top of [Gnat](https://hex.pm/packages/gnat), the officially supported Elixir client for NATS. * [Let's get Jetstream up and running](./getting_started.md) * [Using Broadway with Jetstream](../guides/broadway.md) * [Pull Consumer API](`Gnat.Jetstream.PullConsumer`) * [Create, update and delete Jetstream streams and consumers via Elixir](../guides/managing.md) ================================================ FILE: lib/gnat/command.ex ================================================ defmodule Gnat.Command do @moduledoc false @newline "\r\n" @hpub "HPUB" @pub "PUB" @sub "SUB" @unsub "UNSUB" def build(:pub, topic, payload, []), do: [@pub, " ", topic, " #{IO.iodata_length(payload)}", @newline, payload, @newline] def build(:pub, topic, payload, reply_to: reply), do: [ @pub, " ", topic, " ", reply, " #{IO.iodata_length(payload)}", @newline, payload, @newline ] def build(:pub, topic, payload, headers: headers) do # it takes 10 bytes to add the nats header version line # and 2 more for the newline between headers and payload header_len = IO.iodata_length(headers) + 12 total_len = IO.iodata_length(payload) + header_len [ @hpub, " ", topic, " ", Integer.to_string(header_len), " ", Integer.to_string(total_len), "\r\nNATS/1.0\r\n", headers, @newline, payload, @newline ] end def build(:pub, topic, payload, headers: headers, reply_to: reply) do # it takes 10 bytes to add the nats header version line # and 2 more for the newline between headers and payload header_len = IO.iodata_length(headers) + 12 total_len = IO.iodata_length(payload) + header_len [ @hpub, " ", topic, " ", reply, " ", Integer.to_string(header_len), " ", Integer.to_string(total_len), "\r\nNATS/1.0\r\n", headers, @newline, payload, @newline ] end def build(:sub, topic, sid, []), do: [@sub, " ", topic, " ", Integer.to_string(sid), @newline] def build(:sub, topic, sid, queue_group: qg), do: [@sub, " ", topic, " ", qg, " ", Integer.to_string(sid), @newline] def build(:unsub, sid, []), do: [@unsub, " #{sid}", @newline] def build(:unsub, sid, max_messages: max), do: [@unsub, " #{sid}", " #{max}", @newline] end ================================================ FILE: lib/gnat/connection_supervisor.ex ================================================ defmodule Gnat.ConnectionSupervisor do use GenServer require Logger @moduledoc """ A process that can supervise a named connection for you If you would like to supervise a Gnat connection and have it automatically re-connect in case of failure you can use this module in your supervision tree. It takes a map with the following data: ``` gnat_supervisor_settings = %{ name: :gnat, # (required) the registered named you want to give the Gnat connection backoff_period: 4_000, # number of milliseconds to wait between consecutive reconnect attempts (default: 2_000) connection_settings: [ %{host: '10.0.0.100', port: 4222}, %{host: '10.0.0.101', port: 4222}, ] } ``` The connection settings can specify all of the same values that you pass to `Gnat.start_link/1`. Each time a connection is attempted we will use one of the provided connection settings to open the connection. This is a simplistic way of load balancing your connections across a cluster of nats nodes and allowing failover to other nodes in the cluster if one goes down. To use this in your supervision tree add an entry like this: ``` import Supervisor.Spec worker(Gnat.ConnectionSupervisor, [gnat_supervisor_settings, [name: :my_connection_supervisor]]) ``` The second argument is used as GenServer options so you can give the supervisor a registered name as well if you like. Now in the rest of your code you can call things like: ``` :ok = Gnat.pub(:gnat, "subject", "message") ``` And it will use your supervised connection. If the connection is down when you call that function (or dies during that function) it will raise an error. """ @spec start_link(map(), keyword()) :: GenServer.on_start() def start_link(settings, options \\ []) do GenServer.start_link(__MODULE__, settings, options) end @impl GenServer def init(options) do state = %{ backoff_period: Map.get(options, :backoff_period, 2000), connection_settings: Map.fetch!(options, :connection_settings), name: Map.fetch!(options, :name), gnat: nil } Process.flag(:trap_exit, true) send(self(), :attempt_connection) {:ok, state} end @impl GenServer def handle_info(:attempt_connection, state) do connection_config = random_connection_config(state) Logger.debug("connecting to #{inspect(connection_config)}") case Gnat.start_link(connection_config, name: state.name) do {:ok, gnat} -> {:noreply, %{state | gnat: gnat}} {:error, err} -> Logger.error("failed to connect #{inspect(err)}") Process.send_after(self(), :attempt_connection, state.backoff_period) {:noreply, %{state | gnat: nil}} end end # in OTP 25 and below, we will get back an EXIT message in addition to receiving the {:error, reason} # tuple on from the start_link call above. So if we get an exit message when there is no connection tracked # it means will have already scheduled a new attempt_connection def handle_info({:EXIT, _pid, _reason}, %{gnat: nil} = state) do {:noreply, state} end def handle_info({:EXIT, _pid, reason}, state) do Logger.error("connection failed #{inspect(reason)}") send(self(), :attempt_connection) {:noreply, state} end def handle_info(msg, state) do Logger.error("#{__MODULE__} received unexpected message #{inspect(msg)}") {:noreply, state} end defp random_connection_config(%{connection_settings: connection_settings}) do connection_settings |> Enum.random() end end ================================================ FILE: lib/gnat/consumer_supervisor.ex ================================================ defmodule Gnat.ConsumerSupervisor do use GenServer require Logger alias Gnat.Services.Service @moduledoc """ A process that can supervise consumers for you If you want to subscribe to a few topics and have that subscription last across restarts for you, then this worker can be of help. It also spawns a supervised `Task` for each message it receives. This way errors in message processing don't crash the consumers, but you will still get SASL reports that you can send to services like honeybadger. To use this just add an entry to your supervision tree like this: ``` consumer_supervisor_settings = %{ connection_name: :name_of_supervised_connection, module: MyApp.Server, # a module that implements the Gnat.Server behaviour subscription_topics: [ %{topic: "rpc.MyApp.search", queue_group: "rpc.MyApp.search"}, %{topic: "rpc.MyApp.create", queue_group: "rpc.MyApp.create"}, ], } worker(Gnat.ConsumerSupervisor, [consumer_supervisor_settings, [name: :rpc_consumer]], shutdown: 30_000) ``` The second argument is a keyword list that gets used as the GenServer options so you can pass a name that you want to register for the consumer process if you like. The `:consuming_function` specifies which module and function to call when messages arrive. The function will be called with a single argument which is a `t:Gnat.message/0` just like you get when you call `Gnat.sub/4` directly. You can have a single consumer that subscribes to multiple topics or multiple consumers that subscribe to different topics and call different consuming functions. It is recommended that your `ConsumerSupervisor`s are present later in your supervision tree than your `ConnectionSupervisor`. That way during a shutdown the `ConsumerSupervisor` can attempt a graceful shutdown of the consumer before shutting down the connection. If you want this consumer supervisor to host a NATS service, then you can specify a module that implements the `Gnat.Services.Server` behavior. You'll need to specify the `service_definition` field in the consumer supervisor settings and conforms to the `Gnat.Services.Server.service_configuration` type. Here is an example of configuring the consumer supervisor to manage a service: ``` consumer_supervisor_settings = %{ connection_name: :name_of_supervised_connection, module: MyApp.Service, # a module that implements the Gnat.Services.Server behaviour service_definition: %{ name: "exampleservice", description: "This is an example service", version: "0.1.0", endpoints: [ %{ name: "add", group_name: "calc", }, %{ name: "sub", group_name: "calc" } ] } } worker(Gnat.ConsumerSupervisor, [consumer_supervisor_settings, [name: :myservice_consumer]], shutdown: 30_000) ``` It's also possible to pass a `%{consuming_function: {YourModule, :your_function}}` rather than a `:module` in your settings. In that case no error handling or replying is taking care of for you, microservices cannot be used, and it will be up to your function to take whatever action you want with each message. """ @spec start_link(map(), keyword()) :: GenServer.on_start() def start_link(settings, options \\ []) do GenServer.start_link(__MODULE__, settings, options) end @impl GenServer def init(settings) do Process.flag(:trap_exit, true) {:ok, task_supervisor_pid} = Task.Supervisor.start_link() connection_name = Map.get(settings, :connection_name) subscription_topics = Map.get(settings, :subscription_topics) state = %{ connection_name: connection_name, connection_pid: nil, svc_responder_pid: nil, status: :disconnected, subscription_topics: subscription_topics, subscriptions: [], task_supervisor_pid: task_supervisor_pid } with {:ok, state} <- maybe_append_service(state, settings), {:ok, state} <- maybe_append_module(state, settings), {:ok, state} <- maybe_append_consuming_function(state, settings), :ok <- validate_state(state) do send(self(), :connect) {:ok, state} end end @impl GenServer def handle_info(:connect, %{connection_name: name} = state) do case Process.whereis(name) do nil -> Process.send_after(self(), :connect, 2_000) {:noreply, state} connection_pid -> _ref = Process.monitor(connection_pid) subscriptions = subscribe_to_topics(state, connection_pid) {:noreply, %{ state | status: :connected, connection_pid: connection_pid, subscriptions: subscriptions }} end end def handle_info( {:DOWN, _ref, :process, connection_pid, _reason}, %{connection_pid: connection_pid} = state ) do Process.send_after(self(), :connect, 2_000) {:noreply, %{state | status: :disconnected, connection_pid: nil, subscriptions: []}} end # Ignore DOWN and task result messages from the spawned tasks def handle_info({:DOWN, _ref, :process, _task_pid, _reason}, state), do: {:noreply, state} def handle_info({ref, _result}, state) when is_reference(ref), do: {:noreply, state} def handle_info( {:EXIT, supervisor_pid, _reason}, %{task_supervisor_pid: supervisor_pid} = state ) do {:ok, task_supervisor_pid} = Task.Supervisor.start_link() {:noreply, Map.put(state, :task_supervisor_pid, task_supervisor_pid)} end def handle_info({:msg, gnat_message}, %{service: service, module: module} = state) do Task.Supervisor.async_nolink(state.task_supervisor_pid, Gnat.Services.Server, :execute, [ module, gnat_message, service ]) {:noreply, state} end def handle_info({:msg, gnat_message}, %{module: module} = state) do Task.Supervisor.async_nolink(state.task_supervisor_pid, Gnat.Server, :execute, [ module, gnat_message ]) {:noreply, state} end def handle_info({:msg, gnat_message}, %{consuming_function: {mod, fun}} = state) do Task.Supervisor.async_nolink(state.task_supervisor_pid, mod, fun, [gnat_message]) {:noreply, state} end def handle_info(other, state) do Logger.error("#{__MODULE__} received unexpected message #{inspect(other)}") {:noreply, state} end @impl GenServer def terminate(:shutdown, state) do Logger.info("#{__MODULE__} starting graceful shutdown") Enum.each(state.subscriptions, fn subscription -> :ok = Gnat.unsub(state.connection_pid, subscription) end) # wait for final messages from broker Process.sleep(500) receive_final_broker_messages(state) wait_for_empty_task_supervisor(state) Logger.info("#{__MODULE__} finished graceful shutdown") end def terminate(reason, _state) do Logger.error("#{__MODULE__} unexpected shutdown #{inspect(reason)}") end defp receive_final_broker_messages(state) do receive do info -> handle_info(info, state) receive_final_broker_messages(state) after 0 -> :done end end defp wait_for_empty_task_supervisor(%{task_supervisor_pid: pid} = state) do case Task.Supervisor.children(pid) do [] -> :ok children -> Logger.info("#{__MODULE__}\t\t#{Enum.count(children)} tasks remaining") Process.sleep(1_000) wait_for_empty_task_supervisor(state) end end defp subscribe_to_topics(%{service: service}, connection_pid) do Service.subscription_topics_with_queue_group(service) |> Enum.map(fn {topic, nil} -> {:ok, subscription} = Gnat.sub(connection_pid, self(), topic) subscription {topic, queue_group} -> {:ok, subscription} = Gnat.sub(connection_pid, self(), topic, queue_group: queue_group) subscription end) end defp subscribe_to_topics(state, connection_pid) do Enum.map(state.subscription_topics, fn topic_and_queue_group -> topic = Map.fetch!(topic_and_queue_group, :topic) {:ok, subscription} = case Map.get(topic_and_queue_group, :queue_group) do nil -> Gnat.sub(connection_pid, self(), topic) queue_group -> Gnat.sub(connection_pid, self(), topic, queue_group: queue_group) end subscription end) end defp maybe_append_service(state, %{service_definition: config}) do case Service.init(config) do {:ok, service} -> {:ok, Map.put(state, :service, service)} {:error, errors} -> {:stop, "Invalid service configuration: #{Enum.join(errors, ",")}"} end end defp maybe_append_service(state, _), do: {:ok, state} defp maybe_append_module(state, %{module: module}) do {:ok, Map.put(state, :module, module)} end defp maybe_append_module(state, _), do: {:ok, state} defp maybe_append_consuming_function(state, %{consuming_function: consuming_function}) do {:ok, Map.put(state, :consuming_function, consuming_function)} end defp maybe_append_consuming_function(state, _), do: {:ok, state} defp validate_state(state) do partial = Map.take(state, [:module, :consuming_function]) case Enum.count(partial) do 0 -> {:stop, "You must provide a module or consuming function for the consumer supervisor"} 1 -> :ok _ -> {:stop, "You cannot provide both a module and consuming function. Please specify one or the other."} end end end ================================================ FILE: lib/gnat/handshake.ex ================================================ defmodule Gnat.Handshake do @moduledoc false alias Gnat.Parsec @doc """ This function handles all of the variations of establishing a connection to a nats server and just returns {:ok, socket} or {:error, reason} """ def connect(settings) do host = settings.host |> to_charlist case :gen_tcp.connect(host, settings.port, settings.tcp_opts, settings.connection_timeout) do {:ok, tcp} -> perform_handshake(tcp, settings) result -> result end end def negotiate_settings(server_settings, user_settings) do auth_required = server_settings[:auth_required] || user_settings[:auth_required] || false %{verbose: false} |> negotiate_auth(server_settings, user_settings, auth_required) |> negotiate_headers(server_settings, user_settings) |> negotiate_no_responders(server_settings, user_settings) end defp perform_handshake(tcp, user_settings) do receive do {:tcp, ^tcp, operation} -> {_, [{:info, server_settings}]} = Parsec.parse(Parsec.new(), operation) {:ok, socket} = upgrade_connection(tcp, user_settings) settings = negotiate_settings(server_settings, user_settings) :ok = send_connect(user_settings, settings, socket) {:ok, socket, server_settings} after 1000 -> {:error, "timed out waiting for info"} end end defp send_connect(%{tls: true}, settings, socket) do :ssl.send(socket, "CONNECT " <> Jason.encode!(settings, maps: :strict) <> "\r\n") end defp send_connect(_, settings, socket) do :gen_tcp.send(socket, "CONNECT " <> Jason.encode!(settings, maps: :strict) <> "\r\n") end defp negotiate_auth( settings, _server, %{username: username, password: password} = _user, true = _auth_required ) do Map.merge(settings, %{user: username, pass: password}) end defp negotiate_auth(settings, _server, %{token: token} = _user, true = _auth_required) do Map.merge(settings, %{auth_token: token}) end defp negotiate_auth( settings, %{nonce: nonce} = _server, %{nkey_seed: seed, jwt: jwt} = _user, true = _auth_required ) do {:ok, nkey} = NKEYS.from_seed(seed) signature = NKEYS.sign(nkey, nonce) |> Base.url_encode64() |> String.replace("=", "") Map.merge(settings, %{sig: signature, protocol: 1, jwt: jwt}) end defp negotiate_auth( settings, %{nonce: nonce} = _server, %{nkey_seed: seed} = _user, true = _auth_required ) do {:ok, nkey} = NKEYS.from_seed(seed) signature = NKEYS.sign(nkey, nonce) |> Base.url_encode64() |> String.replace("=", "") public = NKEYS.public_nkey(nkey) Map.merge(settings, %{sig: signature, protocol: 1, nkey: public}) end defp negotiate_auth(settings, _server, _user, _auth_required) do settings end defp negotiate_headers(settings, %{headers: true} = _server, user_settings) do if Map.get(user_settings, :headers, true) do Map.put(settings, :headers, true) else Map.put(settings, :headers, false) end end defp negotiate_headers(_settings, _server, %{headers: true} = _user) do raise "NATS Server does not support headers, but your connection settings specify header support" end defp negotiate_headers(settings, _server, _user) do settings end defp negotiate_no_responders(%{headers: true} = settings, _server_settings, %{ no_responders: true }) do Map.put(settings, :no_responders, true) end defp negotiate_no_responders(settings, _server_settings, _user_settings) do settings end defp upgrade_connection(tcp, %{tls: true, ssl_opts: opts}) do :ok = :inet.setopts(tcp, active: true) :ssl.connect(tcp, opts, 1_000) end defp upgrade_connection(tcp, _settings), do: {:ok, tcp} end ================================================ FILE: lib/gnat/jetstream/api/consumer.ex ================================================ defmodule Gnat.Jetstream.API.Consumer do @moduledoc """ A module representing a NATS JetStream Consumer. Learn more about consumers: https://docs.nats.io/nats-concepts/jetstream/consumers ## The Jetstream.API.Consumer struct The struct's only mandatory field to set is the `:stream_name`. The rest will have the NATS default values set. Note that consumers are ephemeral by default. Set the `:durable_name` to make it durable. Consumer struct fields explanation: * `:stream_name` - name of a stream the consumer is pointing at. * `:domain` - JetStream domain the stream is on. * `:ack_policy` - how the messages should be acknowledged. It has the following options: - `:explicit` - the default policy. It means that each individual message must be acknowledged. It is the only allowed option for pull consumers. - `:none` - no need to ack messages, the server will assume ack on delivery. - `:all` - only the last received message needs to be acked, all the previous messages received are automatically acknowledged. * `:ack_wait` - time in nanoseconds that server will wait for an ack for any individual. If an ack is not received in time, the message will be redelivered. * `:backoff` - list of durations that represents a retry timescale for NAK'd messages or those being normally retried. * `:deliver_group` - when set, will only deliver messages to subscriptions matching that group. * `:deliver_policy` - specifies where in the stream it wants to start receiving messages. It has the following options: - `:all` - the default policy. The consumer will start receiving from the earliest available message. - `:last` - the consumer will start receiving messages with the last message added to the stream. - `:new` - the consumer will only start receiving messages that were created after the customer was created. - `:by_start_sequence` - the consumer is required to specify `:opt_start_seq`, the sequence number to start on. It will receive the closest available message moving forward in the sequence should the message specified have been removed based on the stream limit policy. - `:by_start_time` - the consumer will start with messages on or after this time. The consumer is required to specify `:opt_start_time`, the time in the stream to start at. - `:last_per_subject` - the consumer will start with the latest one for each filtered subject currently in the stream. * `:deliver_subject` - the subject to deliver observed messages. Not allowed for pull subscriptions. A delivery subject is required for queue subscribing as it configures a subject that all the queue consumers should listen on. * `:description` - a short description of the purpose of this customer. * `:durable_name` - the name of the consumer, which the server will track, allowing resuming consumption where left off. By default, a consumer is ephemeral. To make the consumer durable, set the name. See [naming](https://docs.nats.io/running-a-nats-service/nats_admin/jetstream_admin/naming). * `:filter_subject` - when consuming from a stream with a wildcard subject, this allows you to select a subset of the full wildcard subject to receive messages from. * `:flow_control` - when set to true, an empty message with Status header 100 and a reply subject will be sent. Consumers must reply to these messages to control the rate of message delivery. * `:headers_only` - delivers only the headers of messages in the stream and not the bodies. Additionally adds the Nats-Msg-Size header to indicate the size of the removed payload. * `:idle_heartbeat` - if set, the server will regularly send a status message to the client while there are no new messages to send. This lets the client know that the JetStream service is still up and running, even when there is no activity on the stream. The message status header will have a code of 100. Unlike `:flow_control`, it will have no reply to address. It may have a description like "Idle Heartbeat". * `:inactive_threshold` - duration that instructs the server to clean up ephemeral consumers that are inactive for that long. * `:max_ack_pending` - it sets the maximum number of messages without an acknowledgement that can be outstanding, once this limit is reached, message delivery will be suspended. It cannot be used with `:ack_none` ack policy. This maximum number of pending acks applies for all the consumer's subscriber processes. A value of -1 means there can be any number of pending acks (i.e. no flow control). * `:max_batch` - the largest batch property that may be specified when doing a pull on a Pull consumer. * `:max_deliver` - the maximum number of times a specific message will be delivered. Applies to any message that is re-sent due to ack policy. * `:max_expires` - the maximum expires value that may be set when doing a pull on a Pull consumer. * `:max_waiting` - the number of pulls that can be outstanding on a pull consumer, pulls received after this is reached are ignored. * `:opt_start_seq` - use with `:deliver_policy` set to `:by_start_sequence`. It represents the sequence number to start consuming on. * `:opt_start_time` - use with `:deliver_policy` set to `:by_start_time`. It represents the time to start consuming at. * `:rate_limit_bps` - used to throttle the delivery of messages to the consumer, in bits per second. * `:replay_policy` - it applies when the `:deliver_policy` is set to `:all`, `:by_start_sequence` or `:by_start_time`. It has the following options: - `:instant` - the default policy. The messages will be pushed to the client as fast as possible. - `:original` - the messages in the stream will be pushed to the client at the same rate that they were originally received. * `:sample_freq` - Sets the percentage of acknowledgements that should be sampled for observability, 0-100. This value is a binary and for example allows both `30` and `30%` as valid values. """ import Gnat.Jetstream.API.Util @enforce_keys [:stream_name] defstruct [ :backoff, :deliver_group, :deliver_subject, :description, :domain, :durable_name, :filter_subject, :flow_control, :headers_only, :idle_heartbeat, :inactive_threshold, :max_batch, :max_expires, :max_waiting, :opt_start_seq, :opt_start_time, :rate_limit_bps, :sample_freq, :stream_name, ack_policy: :explicit, ack_wait: 30_000_000_000, deliver_policy: :all, max_ack_pending: 20_000, max_deliver: -1, replay_policy: :instant ] @type t :: %__MODULE__{ stream_name: binary(), domain: nil | binary(), ack_policy: :none | :all | :explicit, ack_wait: nil | non_neg_integer(), backoff: nil | [non_neg_integer()], deliver_group: nil | binary(), deliver_policy: :all | :last | :new | :by_start_sequence | :by_start_time | :last_per_subject, deliver_subject: nil | binary(), description: nil | binary(), durable_name: nil | binary(), filter_subject: nil | binary(), flow_control: nil | boolean(), headers_only: nil | boolean(), idle_heartbeat: nil | non_neg_integer(), inactive_threshold: nil | non_neg_integer(), max_ack_pending: nil | integer(), max_batch: nil | integer(), max_deliver: nil | integer(), max_expires: nil | non_neg_integer(), max_waiting: nil | integer(), opt_start_seq: nil | non_neg_integer(), opt_start_time: nil | DateTime.t(), rate_limit_bps: nil | non_neg_integer(), replay_policy: :instant | :original, sample_freq: nil | binary() } @type info :: %{ ack_floor: %{ consumer_seq: non_neg_integer(), stream_seq: non_neg_integer() }, cluster: nil | %{ optional(:name) => binary(), optional(:leader) => binary(), optional(:replicas) => [ %{ :active => non_neg_integer(), :current => boolean(), :name => binary(), optional(:lag) => non_neg_integer(), optional(:offline) => boolean() } ] }, config: config(), created: DateTime.t(), delivered: %{ consumer_seq: non_neg_integer(), stream_seq: non_neg_integer() }, name: binary(), num_ack_pending: non_neg_integer(), num_pending: non_neg_integer(), num_redelivered: non_neg_integer(), num_waiting: non_neg_integer(), push_bound: nil | boolean(), stream_name: binary() } @type config :: %{ ack_policy: :none | :all | :explicit, ack_wait: nil | non_neg_integer(), backoff: nil | [non_neg_integer()], deliver_group: nil | binary(), deliver_policy: :all | :last | :new | :by_start_sequence | :by_start_time | :last_per_subject, deliver_subject: nil | binary(), description: nil | binary(), durable_name: nil | binary(), filter_subject: nil | binary(), flow_control: nil | boolean(), headers_only: nil | boolean(), idle_heartbeat: nil | non_neg_integer(), inactive_threshold: nil | non_neg_integer(), max_ack_pending: nil | integer(), max_batch: nil | integer(), max_deliver: nil | integer(), max_expires: nil | non_neg_integer(), max_waiting: nil | integer(), opt_start_seq: nil | non_neg_integer(), opt_start_time: nil | DateTime.t(), rate_limit_bps: nil | non_neg_integer(), replay_policy: :instant | :original, sample_freq: nil | binary() } @type consumers :: %{ consumers: list(binary()), limit: non_neg_integer(), offset: non_neg_integer(), total: non_neg_integer() } @doc """ Creates a consumer. When consumer's `:durable_name` field is not set, the function creates an ephemeral consumer. Otherwise, it creates a durable consumer. ## Examples iex> {:ok, _response} = Gnat.Jetstream.API.Stream.create(:gnat, %Gnat.Jetstream.API.Stream{name: "astream", subjects: ["subject"]}) iex> {:ok, %{name: "consumer", stream_name: "astream"}} = Gnat.Jetstream.API.Consumer.create(:gnat, %Gnat.Jetstream.API.Consumer{durable_name: "consumer", stream_name: "astream"}) iex> {:ok, _response} = Gnat.Jetstream.API.Stream.create(:gnat, %Gnat.Jetstream.API.Stream{name: "astream", subjects: ["subject"]}) iex> {:error, %{"description" => "consumer delivery policy is deliver by start sequence, but optional start sequence is not set"}} = Gnat.Jetstream.API.Consumer.create(:gnat, %Gnat.Jetstream.API.Consumer{durable_name: "consumer", stream_name: "astream", deliver_policy: :by_start_sequence}) """ @spec create(conn :: Gnat.t(), consumer :: t()) :: {:ok, info()} | {:error, term()} def create(conn, %__MODULE__{durable_name: name} = consumer) when not is_nil(name) do create_topic = "#{js_api(consumer.domain)}.CONSUMER.DURABLE.CREATE.#{consumer.stream_name}.#{name}" with :ok <- validate_durable(consumer), {:ok, raw_response} <- request(conn, create_topic, create_payload(consumer)) do {:ok, to_info(raw_response)} end end def create(conn, %__MODULE__{} = consumer) do create_topic = "#{js_api(consumer.domain)}.CONSUMER.CREATE.#{consumer.stream_name}" with :ok <- validate(consumer), {:ok, raw_response} <- request(conn, create_topic, create_payload(consumer)) do {:ok, to_info(raw_response)} end end @doc """ Deletes a consumer. ## Examples iex> {:ok, _response} = Gnat.Jetstream.API.Stream.create(:gnat, %Gnat.Jetstream.API.Stream{name: "astream", subjects: ["subject"]}) iex> {:ok, _response} = Gnat.Jetstream.API.Consumer.create(:gnat, %Gnat.Jetstream.API.Consumer{durable_name: "consumer", stream_name: "astream"}) iex> Gnat.Jetstream.API.Consumer.delete(:gnat, "astream", "consumer") :ok iex> {:error, %{"code" => 404, "description" => "stream not found"}} = Gnat.Jetstream.API.Consumer.delete(:gnat, "wrong_stream", "consumer") """ @spec delete( conn :: Gnat.t(), stream_name :: binary(), consumer_name :: binary(), domain :: nil | binary() ) :: :ok | {:error, any()} def delete(conn, stream_name, consumer_name, domain \\ nil) do topic = "#{js_api(domain)}.CONSUMER.DELETE.#{stream_name}.#{consumer_name}" with {:ok, _response} <- request(conn, topic, "") do :ok end end @doc """ Information about the consumer. ## Examples iex> {:ok, _response} = Gnat.Jetstream.API.Stream.create(:gnat, %Gnat.Jetstream.API.Stream{name: "astream", subjects: ["subject"]}) iex> {:ok, _response} = Gnat.Jetstream.API.Consumer.create(:gnat, %Gnat.Jetstream.API.Consumer{durable_name: "consumer", stream_name: "astream"}) iex> {:ok, %{created: _}} = Gnat.Jetstream.API.Consumer.info(:gnat, "astream", "consumer") iex> {:error, %{"code" => 404, "description" => "stream not found"}} = Gnat.Jetstream.API.Consumer.info(:gnat, "wrong_stream", "consumer") """ @spec info( conn :: Gnat.t(), stream_name :: binary(), consumer_name :: binary(), domain :: nil | binary() ) :: {:ok, info()} | {:error, any()} def info(conn, stream_name, consumer_name, domain \\ nil) do topic = "#{js_api(domain)}.CONSUMER.INFO.#{stream_name}.#{consumer_name}" with {:ok, raw} <- request(conn, topic, "") do {:ok, to_info(raw)} end end @doc """ Paged list of known consumers, including their current info. ## Examples iex> {:ok, _response} = Gnat.Jetstream.API.Stream.create(:gnat, %Gnat.Jetstream.API.Stream{name: "astream", subjects: ["subject"]}) iex> {:ok, %{consumers: _, limit: 1024, offset: 0, total: _}} = Gnat.Jetstream.API.Consumer.list(:gnat, "astream") iex> {:error, %{"code" => 404, "description" => "stream not found"}} = Gnat.Jetstream.API.Consumer.list(:gnat, "wrong_stream") """ @spec list( conn :: Gnat.t(), stream_name :: binary(), params :: [offset: non_neg_integer(), domain: nil | binary()] ) :: {:ok, consumers()} | {:error, term()} def list(conn, stream_name, params \\ []) do domain = Keyword.get(params, :domain) payload = Jason.encode!(%{ offset: Keyword.get(params, :offset, 0) }) with {:ok, raw} <- request(conn, "#{js_api(domain)}.CONSUMER.NAMES.#{stream_name}", payload) do response = %{ consumers: Map.get(raw, "consumers"), limit: Map.get(raw, "limit"), offset: Map.get(raw, "offset"), total: Map.get(raw, "total") } {:ok, response} end end @doc """ Requests a next message from a stream to be consumed. The response (consumed message)will be sent on the subject given as the `reply_to` parameter. ## Options * `batch` - How many messages to receive. Messages will be sent to the `reply_to` subject separately. Defaults to 1. * `expires` - Time in nanoseconds the request will be kept in the server. Once this time passes a message with empty body and topic set to `reply_to` subject is sent. Useful when polling the server frequently and not wanting the pull requests to accumulate. By default, the pull request stays in the server until a message comes. * `no_wait` - Boolean value which indicates whether the pull request should be accumulated on the server. When set to true and no message is present to be consumed, a message with empty body and topic value set to `reply_to` is sent. Defaults to false. ## Example iex> {:ok, _response} = Gnat.Jetstream.API.Stream.create(:gnat, %Gnat.Jetstream.API.Stream{name: "astream", subjects: ["subject"]}) iex> {:ok, _response} = Gnat.Jetstream.API.Consumer.create(:gnat, %Gnat.Jetstream.API.Consumer{durable_name: "consumer", stream_name: "astream"}) iex> {:ok, _sid} = Gnat.sub(:gnat, self(), "reply_subject") iex> :ok = Gnat.Jetstream.API.Consumer.request_next_message(:gnat, "astream", "consumer", "reply_subject") iex> :ok = Gnat.pub(:gnat, "subject", "message1") iex> assert_receive {:msg, %{body: "message1", topic: "subject"}} """ @spec request_next_message( conn :: Gnat.t(), stream_name :: binary(), consumer_name :: binary(), reply_to :: String.t(), domain :: nil | binary(), opts :: keyword() ) :: :ok def request_next_message( conn, stream_name, consumer_name, reply_to, domain \\ nil, opts \\ [] ) do default_payload = %{batch: 1} put_option_if_not_nil = fn payload, option_key -> if option_value = opts[option_key] do Map.put(payload, option_key, option_value) else payload end end payload = default_payload |> put_option_if_not_nil.(:batch) |> put_option_if_not_nil.(:no_wait) |> put_option_if_not_nil.(:expires) |> Jason.encode!() Gnat.pub( conn, "#{js_api(domain)}.CONSUMER.MSG.NEXT.#{stream_name}.#{consumer_name}", payload, reply_to: reply_to ) end # https://docs.nats.io/running-a-nats-service/configuration/leafnodes/jetstream_leafnodes defp js_api(nil), do: "$JS.API" defp js_api(""), do: "$JS.API" defp js_api(domain), do: "$JS.#{domain}.API" defp create_payload(%__MODULE__{} = cons) do %{ config: %{ ack_policy: cons.ack_policy, ack_wait: cons.ack_wait, backoff: cons.backoff, deliver_group: cons.deliver_group, deliver_policy: cons.deliver_policy, deliver_subject: cons.deliver_subject, description: cons.description, durable_name: cons.durable_name, filter_subject: cons.filter_subject, flow_control: cons.flow_control, headers_only: cons.headers_only, idle_heartbeat: cons.idle_heartbeat, inactive_threshold: cons.inactive_threshold, max_ack_pending: cons.max_ack_pending, max_batch: cons.max_batch, max_deliver: cons.max_deliver, max_expires: cons.max_expires, max_waiting: cons.max_waiting, opt_start_seq: cons.opt_start_seq, opt_start_time: cons.opt_start_time, rate_limit_bps: cons.rate_limit_bps, replay_policy: cons.replay_policy, sample_freq: cons.sample_freq }, stream_name: cons.stream_name } |> Jason.encode!() end defp to_config(raw) do %{ ack_policy: raw |> Map.get("ack_policy") |> to_sym(), ack_wait: raw |> Map.get("ack_wait"), backoff: Map.get(raw, "backoff"), deliver_group: Map.get(raw, "deliver_group"), deliver_policy: raw |> Map.get("deliver_policy") |> to_sym(), deliver_subject: raw |> Map.get("deliver_subject"), description: Map.get(raw, "description"), durable_name: Map.get(raw, "durable_name"), filter_subject: raw |> Map.get("filter_subject"), flow_control: Map.get(raw, "flow_control"), headers_only: Map.get(raw, "headers_only"), idle_heartbeat: Map.get(raw, "idle_heartbeat"), inactive_threshold: Map.get(raw, "inactive_threshold"), max_ack_pending: Map.get(raw, "max_ack_pending"), max_batch: Map.get(raw, "max_batch"), max_deliver: Map.get(raw, "max_deliver"), max_expires: Map.get(raw, "max_expires"), max_waiting: Map.get(raw, "max_waiting"), opt_start_seq: raw |> Map.get("opt_start_seq"), opt_start_time: raw |> Map.get("opt_start_time") |> to_datetime(), rate_limit_bps: Map.get(raw, "rate_limit_bps"), replay_policy: raw |> Map.get("replay_policy") |> to_sym(), sample_freq: Map.get(raw, "sample_freq") } end defp to_info(raw) do %{ ack_floor: %{ consumer_seq: get_in(raw, ["ack_floor", "consumer_seq"]), stream_seq: get_in(raw, ["ack_floor", "stream_seq"]) }, cluster: Map.get(raw, "cluster"), config: to_config(Map.get(raw, "config")), created: raw |> Map.get("created") |> to_datetime(), delivered: %{ consumer_seq: get_in(raw, ["delivered", "consumer_seq"]), stream_seq: get_in(raw, ["delivered", "stream_seq"]) }, name: Map.get(raw, "name"), num_ack_pending: Map.get(raw, "num_ack_pending"), num_pending: Map.get(raw, "num_pending"), num_redelivered: Map.get(raw, "num_redelivered"), num_waiting: Map.get(raw, "num_waiting"), push_bound: Map.get(raw, "push_bound"), stream_name: Map.get(raw, "stream_name") } end defp validate(consumer) do cond do consumer.stream_name == nil -> {:error, "must have a :stream_name set"} is_binary(consumer.stream_name) == false -> {:error, "stream_name must be a string"} valid_name?(consumer.stream_name) == false -> {:error, "invalid stream_name: " <> invalid_name_message()} consumer.deliver_policy not in [ :all, :last, :new, :by_start_sequence, :by_start_time, :last_per_subject ] -> {:error, "invalid deliver policy: #{consumer.deliver_policy}"} consumer.replay_policy not in [:instant, :original] -> {:error, "invalid replay policy: #{consumer.replay_policy}"} true -> :ok end end defp validate_durable(consumer) do with :ok <- validate(consumer) do cond do is_binary(consumer.durable_name) == false -> {:error, "durable_name must be a string"} valid_name?(consumer.durable_name) == false -> {:error, "invalid durable_name: " <> invalid_name_message()} true -> :ok end end end end ================================================ FILE: lib/gnat/jetstream/api/kv/entry.ex ================================================ defmodule Gnat.Jetstream.API.KV.Entry do @moduledoc """ A parsed view of a single message from a Key/Value bucket's underlying stream. Messages delivered from a KV bucket's stream encode three different operations (put, delete, purge) using a combination of the `kv-operation` header, the `nats-marker-reason` header, and the absence of any headers. Recovering the original key also requires stripping the `$KV..` subject prefix. This module captures that convention in one place so both the built-in `Gnat.Jetstream.API.KV.Watcher` (push consumer) and user-supplied `Gnat.Jetstream.PullConsumer` implementations can share it. ## Using with a custom PullConsumer A common use case is hydrating a local cache from a KV bucket by driving a `Gnat.Jetstream.PullConsumer`. Inside `c:handle_message/2`, convert the raw message into an `Entry` and branch on the operation: defmodule MyApp.KVCache do use Gnat.Jetstream.PullConsumer alias Gnat.Jetstream.API.KV @bucket "my_bucket" @impl true def handle_message(message, state) do case KV.Entry.from_message(message, @bucket) do {:ok, %KV.Entry{operation: :put, key: key, value: value}} -> {:ack, put_in(state.cache[key], value)} {:ok, %KV.Entry{operation: op, key: key}} when op in [:delete, :purge] -> {:ack, update_in(state.cache, &Map.delete(&1, key))} :ignore -> {:ack, state} end end end The returned struct also carries the JetStream `revision` (stream sequence), `created` timestamp, and `delta` (`num_pending`) when the message includes JetStream metadata, which is useful for detecting when the consumer has caught up with the tail of the stream (`delta == 0`). ## Messages that are not KV records `from_message/2` returns `:ignore` when the input is not a KV record — for example a JetStream status message (`100` heartbeat, `404`/`408` pull terminator, `409` leadership change) or a message whose subject does not belong to the given bucket. In normal operation the `Watcher` and `PullConsumer` layers filter status messages out before they reach user code, so this is a defensive fallback rather than something consumers are expected to rely on. """ alias Gnat.Jetstream.API.Message @operation_header "kv-operation" @operation_del "DEL" @operation_purge "PURGE" @nats_marker_reason_header "nats-marker-reason" @subject_prefix "$KV." @type operation :: :put | :delete | :purge @type t :: %__MODULE__{ bucket: String.t(), key: String.t(), value: binary(), operation: operation(), revision: non_neg_integer() | nil, created: DateTime.t() | nil, delta: non_neg_integer() | nil } defstruct [:bucket, :key, :value, :operation, :revision, :created, :delta] @doc """ Parse a NATS message delivered from a KV bucket's underlying stream into an `Entry`. `bucket_name` must match the bucket the message was published to; it is used to strip the `$KV..` subject prefix and recover the key. Returns `:ignore` if the message is not a KV record for the given bucket (JetStream status message, wrong subject, etc.). The `:revision`, `:created`, and `:delta` fields are populated when the message carries a JetStream `$JS.ACK...` reply subject. For messages without one (e.g. direct get responses), those fields are `nil`. """ @spec from_message(Gnat.message(), bucket_name :: String.t()) :: {:ok, t()} | :ignore def from_message(message, bucket_name) do with false <- status_message?(message), {:ok, key} <- extract_key(message, bucket_name) do entry = %__MODULE__{ bucket: bucket_name, key: key, value: Map.get(message, :body, ""), operation: operation(message) } {:ok, apply_metadata(entry, message)} else _ -> :ignore end end defp status_message?(%{status: status}) when is_binary(status) and status != "", do: true defp status_message?(_), do: false defp extract_key(%{topic: topic}, bucket_name) do prefix = @subject_prefix <> bucket_name <> "." if String.starts_with?(topic, prefix) do {:ok, binary_part(topic, byte_size(prefix), byte_size(topic) - byte_size(prefix))} else :error end end defp operation(%{headers: headers}) when is_list(headers) do Enum.find_value(headers, :put, fn {@operation_header, @operation_del} -> :delete {@operation_header, @operation_purge} -> :purge {@nats_marker_reason_header, _} -> :delete _ -> false end) end defp operation(_message), do: :put defp apply_metadata(entry, message) do case Message.metadata(message) do {:ok, metadata} -> %__MODULE__{ entry | revision: metadata.stream_seq, created: metadata.timestamp, delta: metadata.num_pending } {:error, _} -> entry end end end ================================================ FILE: lib/gnat/jetstream/api/kv/watcher.ex ================================================ defmodule Gnat.Jetstream.API.KV.Watcher do @moduledoc """ The watcher server establishes a subscription to the changes that occur to a given key-value bucket. The consumer-supplied handler function will be sent an indicator as to whether the change is a delete or an add, as well as the key being changed and the value (if it was added). Ensure that you call `stop` with a watcher pid when you no longer need to be notified about key changes """ use GenServer alias Gnat.Jetstream.API.{Consumer, KV, Util} alias Gnat.Jetstream.API.KV.Entry # Matches the ordered-consumer defaults used by the nats.go KV watcher: # a 5s idle heartbeat and server-driven flow control. The flow-control # messages arrive as 100-status messages with a reply subject — the client # is expected to publish an empty reply so the server releases backpressure. @flow_control_heartbeat_ns 5_000_000_000 @type keywatch_handler :: (action :: :key_deleted | :key_added, key :: String.t(), value :: any() -> nil) @type watcher_options :: {:conn, Gnat.t()} | {:bucket_name, String.t()} | {:handler, keywatch_handler()} @spec start_link(opts :: [watcher_options()]) :: GenServer.on_start() def start_link(opts) do GenServer.start_link(__MODULE__, opts) end def stop(pid) do GenServer.stop(pid) end def init(opts) do {:ok, {sub, consumer_name}} = subscribe(opts[:conn], opts[:bucket_name]) {:ok, %{ handler: opts[:handler], conn: opts[:conn], bucket_name: opts[:bucket_name], sub: sub, consumer_name: consumer_name, domain: Keyword.get(opts, :domain) }} end def terminate(_reason, state) do stream = KV.stream_name(state.bucket_name) :ok = Gnat.unsub(state.conn, state.sub) :ok = Consumer.delete(state.conn, stream, state.consumer_name, state.domain) end # Flow-control request: the server is asking us to acknowledge that we're # keeping up. Responding releases backpressure so the server continues # delivering messages to slow handlers rather than dropping us as a slow # consumer. def handle_info({:msg, %{status: "100", reply_to: reply_to}}, state) when is_binary(reply_to) and reply_to != "" do :ok = Gnat.pub(state.conn, reply_to, "") {:noreply, state} end # Idle heartbeat (status 100 with no reply_to) and any other informational # status message (404, 408, 409, etc.) — not a stream record, drop it. def handle_info({:msg, %{status: status}}, state) when is_binary(status) and status != "" do {:noreply, state} end def handle_info({:msg, message}, state) do case Entry.from_message(message, state.bucket_name) do {:ok, entry} -> state.handler.(action(entry.operation), entry.key, entry.value) :ignore -> :ok end {:noreply, state} end defp action(:put), do: :key_added defp action(:delete), do: :key_deleted defp action(:purge), do: :key_purged defp subscribe(conn, bucket_name) do stream = KV.stream_name(bucket_name) inbox = Util.reply_inbox() consumer_name = "all_key_values_watcher_#{Util.nuid()}" with {:ok, sub} <- Gnat.sub(conn, self(), inbox), {:ok, _consumer} <- Consumer.create(conn, %Consumer{ durable_name: consumer_name, deliver_subject: inbox, stream_name: stream, ack_policy: :none, max_ack_pending: -1, max_deliver: 1, flow_control: true, idle_heartbeat: @flow_control_heartbeat_ns }) do {:ok, {sub, consumer_name}} end end end ================================================ FILE: lib/gnat/jetstream/api/kv.ex ================================================ defmodule Gnat.Jetstream.API.KV do @moduledoc """ API for interacting with the Key/Value store functionality in Nats Jetstream. Learn about the Key/Value store: https://docs.nats.io/nats-concepts/jetstream/key-value-store ## Consuming KV changes from a custom PullConsumer `watch/3` covers the common "push consumer" use case for reacting to bucket changes. If you need a pull-based consumer instead (e.g. to hydrate a local cache and detect when it is caught up with the stream), see `Gnat.Jetstream.API.KV.Entry` for a shared helper that parses a raw NATS message into a KV operation/key/value triple with optional revision metadata. """ alias Gnat.Jetstream.API.{Stream} @stream_prefix "KV_" @subject_prefix "$KV." @two_minutes_in_nanoseconds 120_000_000_000 @type bucket_options :: {:history, non_neg_integer()} | {:ttl, non_neg_integer()} | {:limit_marker_ttl, non_neg_integer()} | {:max_bucket_size, non_neg_integer()} | {:max_value_size, non_neg_integer()} | {:description, binary()} | {:replicas, non_neg_integer()} | {:storage, :file | :memory} | {:placement, Stream.placement()} @doc """ Create a new Key/Value bucket. Can include the following options * `:history` - How many historic values to keep per key (defaults to 1, max of 64) * `:ttl` - How long to keep values for (in nanoseconds) * `:limit_marker_ttl` - How long the bucket keeps markers when keys are removed by the TTL setting. * `:max_bucket_size` - The max number of bytes the bucket can hold * `:max_value_size` - The max number of bytes a value may be * `:description` - A description for the bucket * `:replicas` - How many replicas of the data to store * `:storage` - Storage backend to use (:file, :memory) * `:placement` - A map with :cluster (required) and :tags (optional) ## Examples iex> {:ok, info} = Jetstream.API.KV.create_bucket(:gnat, "my_bucket") """ @spec create_bucket(conn :: Gnat.t(), bucket_name :: binary(), params :: [bucket_options()]) :: {:ok, Stream.info()} | {:error, any()} def create_bucket(conn, bucket_name, params \\ []) do # The primary NATS docs don't provide information about how to interact # with Key-Value functionality over the wire. Turns out the KV store is # just a Stream under-the-hood # Discovered these settings from looking at the `nats-server -js -DV` logs # as well as the GoLang implementation https://github.com/nats-io/nats.go/blob/dd91b86bc4f7fa0f061fefe11506aaee413bfafd/kv.go#L339 # If the settings aren't correct, NATS will not consider it a valid KV store stream = %Stream{ name: stream_name(bucket_name), subjects: stream_subjects(bucket_name), description: Keyword.get(params, :description), max_msgs_per_subject: Keyword.get(params, :history, 1), discard: :new, deny_delete: true, allow_rollup_hdrs: true, max_age: Keyword.get(params, :ttl, 0), max_bytes: Keyword.get(params, :max_bucket_size, -1), max_msg_size: Keyword.get(params, :max_value_size, -1), num_replicas: Keyword.get(params, :replicas, 1), storage: Keyword.get(params, :storage, :file), placement: Keyword.get(params, :placement), duplicate_window: adjust_duplicate_window(Keyword.get(params, :ttl, 0)), subject_delete_marker_ttl: Keyword.get(params, :limit_marker_ttl, 0) } Stream.create(conn, stream) end # The `duplicate_window` can't be greater than the `max_age`. The default `duplicate_window` # is 2 minutes. We'll keep the 2 minute window UNLESS the ttl is less than 2 minutes defp adjust_duplicate_window(ttl) when ttl > 0 and ttl < @two_minutes_in_nanoseconds, do: ttl defp adjust_duplicate_window(_ttl), do: @two_minutes_in_nanoseconds @doc """ Delete a Key/Value bucket ## Examples iex> :ok = Jetstream.API.KV.delete_bucket(:gnat, "my_bucket") """ @spec delete_bucket(conn :: Gnat.t(), bucket_name :: binary()) :: :ok | {:error, any()} def delete_bucket(conn, bucket_name) do Stream.delete(conn, stream_name(bucket_name)) end @doc """ Create a Key in a Key/Value Bucket ## Options * `:timeout` - receive timeout for the request ## Examples iex> :ok = Jetstream.API.KV.create_key(:gnat, "my_bucket", "my_key", "my_value") """ @spec create_key( conn :: Gnat.t(), bucket_name :: binary(), key :: binary(), value :: binary(), opts :: keyword() ) :: :ok | {:error, any()} def create_key(conn, bucket_name, key, value, opts \\ []) do timeout = Keyword.get(opts, :timeout, 5_000) reply = Gnat.request(conn, key_name(bucket_name, key), value, receive_timeout: timeout) case reply do {:ok, _} -> :ok error -> error end end @doc """ Delete a Key from a K/V Bucket ## Examples iex> :ok = Jetstream.API.KV.delete_key(:gnat, "my_bucket", "my_key") """ @spec delete_key( conn :: Gnat.t(), bucket_name :: binary(), key :: binary(), opts :: keyword() ) :: :ok | {:error, any()} def delete_key(conn, bucket_name, key, opts \\ []) do timeout = Keyword.get(opts, :timeout, 5_000) reply = Gnat.request(conn, key_name(bucket_name, key), "", headers: [{"KV-Operation", "DEL"}], receive_timeout: timeout ) case reply do {:ok, _} -> :ok error -> error end end @doc """ Purge a Key from a K/V bucket. This will remove any revision history the key had ## Examples iex> :ok = Jetstream.API.KV.purge_key(:gnat, "my_bucket", "my_key") """ @spec purge_key( conn :: Gnat.t(), bucket_name :: binary(), key :: binary(), opts :: keyword() ) :: :ok | {:error, any()} def purge_key(conn, bucket_name, key, opts \\ []) do timeout = Keyword.get(opts, :timeout, 5_000) reply = Gnat.request(conn, key_name(bucket_name, key), "", headers: [{"KV-Operation", "PURGE"}, {"Nats-Rollup", "sub"}], receive_timeout: timeout ) case reply do {:ok, _} -> :ok error -> error end end @doc """ Put a value into a Key in a K/V Bucket ## Examples iex> :ok = Jetstream.API.KV.put_value(:gnat, "my_bucket", "my_key", "my_value") """ @spec put_value( conn :: Gnat.t(), bucket_name :: binary(), key :: binary(), value :: binary(), opts :: keyword() ) :: :ok | {:error, any()} def put_value(conn, bucket_name, key, value, opts \\ []) do timeout = Keyword.get(opts, :timeout, 5_000) reply = Gnat.request(conn, key_name(bucket_name, key), value, receive_timeout: timeout) case reply do {:ok, _} -> :ok error -> error end end @doc """ Get the value for a key in a particular K/V bucket ## Examples iex> "my_value" = Jetstream.API.KV.get_value(:gnat, "my_bucket", "my_key") """ @spec get_value(conn :: Gnat.t(), bucket_name :: binary(), key :: binary()) :: binary() | {:error, any()} | nil def get_value(conn, bucket_name, key) do case Stream.get_message(conn, stream_name(bucket_name), %{ last_by_subj: key_name(bucket_name, key) }) do {:ok, message} -> message.data error -> error end end @doc """ Get all the non-deleted key-value pairs for a Bucket ## Options * `:batch` - Number of messages to fetch per request (default: 250) * `:domain` - JetStream domain (default: nil) ## Examples iex> {:ok, %{"key1" => "value1"}} = Jetstream.API.KV.contents(:gnat, "my_bucket") iex> {:ok, contents} = Jetstream.API.KV.contents(:gnat, "my_bucket", batch: 500) """ @spec contents(conn :: Gnat.t(), bucket_name :: binary(), opts :: keyword()) :: {:ok, map()} | {:error, binary()} def contents(conn, bucket_name, opts \\ []) do alias Gnat.Jetstream.Pager stream = stream_name(bucket_name) domain = Keyword.get(opts, :domain) batch_size = Keyword.get(opts, :batch, 250) pager_opts = [domain: domain, batch: batch_size] Pager.reduce(conn, stream, pager_opts, %{}, fn msg, acc -> case msg do %{topic: key, body: body, headers: headers} -> if {"kv-operation", "DEL"} in headers do acc else Map.put(acc, subject_to_key(key, bucket_name), body) end %{topic: key, body: body} -> Map.put(acc, subject_to_key(key, bucket_name), body) end end) end @doc """ Get all the non-deleted keys for a Bucket ## Options * `:batch` - Number of messages to fetch per request (default: 250) * `:domain` - JetStream domain (default: nil) ## Examples iex> {:ok, ["key1", "key2"]} = Jetstream.API.KV.keys(:gnat, "my_bucket") iex> {:ok, keys} = Jetstream.API.KV.keys(:gnat, "my_bucket", batch: 500) """ @spec keys(conn :: Gnat.t(), bucket_name :: binary(), opts :: keyword()) :: {:ok, list(binary())} | {:error, binary()} def keys(conn, bucket_name, opts \\ []) do alias Gnat.Jetstream.Pager stream = stream_name(bucket_name) domain = Keyword.get(opts, :domain) batch_size = Keyword.get(opts, :batch, 250) pager_opts = [domain: domain, headers_only: true, batch: batch_size] result = Pager.reduce(conn, stream, pager_opts, MapSet.new(), fn msg, acc -> case msg do %{topic: key, headers: headers} -> cond do {"kv-operation", "DEL"} in headers -> MapSet.delete(acc, subject_to_key(key, bucket_name)) {"kv-operation", "PURGE"} in headers -> MapSet.delete(acc, subject_to_key(key, bucket_name)) true -> MapSet.put(acc, subject_to_key(key, bucket_name)) end %{topic: key} -> MapSet.put(acc, subject_to_key(key, bucket_name)) end end) case result do {:ok, key_set} -> {:ok, Enum.sort(MapSet.to_list(key_set))} error -> error end end @doc """ Information about the state of the bucket's Stream. ## Opts * `:domain` - (default `nil`) the domain of the bucket """ @spec info(conn :: Gnat.t(), bucket_name :: binary(), keyword()) :: {:ok, Stream.info()} | {:error, any()} def info(conn, bucket_name, opts \\ []) do Stream.info(conn, @stream_prefix <> bucket_name, Keyword.get(opts, :domain)) end @doc ~S""" Starts a monitor for key changes in a given bucket. Supply a handler that will receive key change notifications. ## Examples iex> {:ok, _pid} = Jetstream.API.KV.watch(:gnat, "my_bucket", fn action, key, value -> ...> IO.puts("#{action} taken on #{key}") ...> end) """ def watch(conn, bucket_name, handler) do Gnat.Jetstream.API.KV.Watcher.start_link( conn: conn, bucket_name: bucket_name, handler: handler ) end @doc ~S""" Stops a previously running monitor. This will unsubscribe from the key changes and remove the ephemeral consumer ## Examples iex> :ok = Jetstream.API.KV.unwatch(pid) """ def unwatch(pid) do Gnat.Jetstream.API.KV.Watcher.stop(pid) end @spec is_kv_bucket_stream?(stream_name :: binary()) :: boolean() @deprecated "Use Gnat.Jetstream.API.KV.kv_bucket_stream?/1 instead" def is_kv_bucket_stream?(stream_name) do kv_bucket_stream?(stream_name) end @doc """ Returns true if the provided stream is a KV bucket, false otherwise ## Parameters * `stream_name` - the stream name to test """ @spec kv_bucket_stream?(stream_name :: binary()) :: boolean() def kv_bucket_stream?(stream_name) do String.starts_with?(stream_name, "KV_") end @doc """ Returns a list of all the buckets in the KV """ @spec list_buckets(conn :: Gnat.t()) :: {:error, term()} | {:ok, list(String.t())} def list_buckets(conn) do with {:ok, %{streams: streams}} <- Stream.list(conn) do stream_names = for bucket <- streams, kv_bucket_stream?(bucket) do String.trim_leading(bucket, @stream_prefix) end {:ok, stream_names} else {:error, reason} -> {:error, reason} end end @doc false def stream_name(bucket_name) do "#{@stream_prefix}#{bucket_name}" end defp stream_subjects(bucket_name) do ["#{@subject_prefix}#{bucket_name}.>"] end defp key_name(bucket_name, key) do "#{@subject_prefix}#{bucket_name}.#{key}" end @doc false def subject_to_key(subject, bucket_name) do String.replace(subject, "#{@subject_prefix}#{bucket_name}.", "") end end ================================================ FILE: lib/gnat/jetstream/api/message.ex ================================================ defmodule Gnat.Jetstream.API.Message do @moduledoc """ This module provides a way to parse the `reply_to` received by a PullConsumer and get some useful information about the state of the consumer. """ # Based on: # https://github.com/nats-io/nats.py/blob/d9f24b4beae541b7723873ba0a786ea7c0ecb3d5/nats/aio/msg.py#L182 defmodule Metadata do @type t :: %__MODULE__{ :stream_seq => integer(), :consumer_seq => integer(), :num_pending => integer(), :num_delivered => integer(), :timestamp => DateTime.t(), :stream => String.t(), :consumer => String.t(), :domain => String.t() | nil } defstruct [ :stream_seq, :consumer_seq, :num_pending, :num_delivered, :timestamp, :stream, :consumer, :domain ] end @spec metadata(message :: Gnat.message()) :: {:ok, Metadata.t()} | {:error, term()} def metadata(%{reply_to: "$JS.ACK." <> ack_topic}), do: decode_reply_to(String.split(ack_topic, ".")) def metadata(_), do: {:error, :no_jetstream_message} # # Subject without domain: # $JS.ACK....... defp decode_reply_to([ stream, consumer, num_delivered, stream_seq, consumer_seq, ts, num_pending ]) do with {ts, ""} <- Integer.parse(ts), {:ok, ts} <- DateTime.from_unix(ts, :nanosecond), {stream_seq, ""} <- Integer.parse(stream_seq), {consumer_seq, ""} <- Integer.parse(consumer_seq), {num_delivered, ""} <- Integer.parse(num_delivered), {num_pending, ""} <- Integer.parse(num_pending) do {:ok, %Metadata{ stream_seq: stream_seq, consumer_seq: consumer_seq, num_delivered: num_delivered, num_pending: num_pending, timestamp: ts, stream: stream, consumer: consumer }} else _ -> {:error, :invalid_ack_reply_to} end end # Subject with domain: # $JS.ACK....... # ... defp decode_reply_to([ domain, _account_hash, stream, consumer, num_delivered, stream_seq, consumer_seq, ts, num_pending, _random_value ]) do case decode_reply_to([ stream, consumer, num_delivered, stream_seq, consumer_seq, ts, num_pending ]) do {:ok, metadata} -> domain = if domain == "_", do: nil, else: domain {:ok, %{metadata | domain: domain}} err = {:error, _} -> err end end defp decode_reply_to(_), do: {:error, :invalid_ack_reply_to} end ================================================ FILE: lib/gnat/jetstream/api/object/meta.ex ================================================ defmodule Gnat.Jetstream.API.Object.Meta do @enforce_keys [:bucket, :chunks, :digest, :name, :nuid, :size] defstruct bucket: nil, chunks: nil, deleted: false, digest: nil, name: nil, nuid: nil, size: nil @type t :: %__MODULE__{ bucket: String.t(), chunks: non_neg_integer(), deleted: boolean(), digest: String.t(), name: String.t(), nuid: String.t(), size: non_neg_integer() } end defimpl Jason.Encoder, for: Gnat.Jetstream.API.Object.Meta do alias Gnat.Jetstream.API.Object.Meta def encode(%Meta{deleted: true} = meta, opts) do Map.take(meta, [:bucket, :chunks, :deleted, :digest, :name, :nuid, :size]) |> Jason.Encode.map(opts) end def encode(meta, opts) do Map.take(meta, [:bucket, :chunks, :digest, :name, :nuid, :size]) |> Jason.Encode.map(opts) end end ================================================ FILE: lib/gnat/jetstream/api/object.ex ================================================ defmodule Gnat.Jetstream.API.Object do @moduledoc """ API for interacting with the JetStream Object Store Learn more about Object Store: https://docs.nats.io/nats-concepts/jetstream/obj_store """ alias Gnat.Jetstream.API.{Consumer, Stream, Util} alias Gnat.Jetstream.API.Object.Meta @stream_prefix "OBJ_" @subject_prefix "$O." @type bucket_opt :: {:description, String.t()} | {:max_bucket_size, integer()} | {:max_chunk_size, integer()} | {:placement, Stream.placement()} | {:replicas, non_neg_integer()} | {:storage, :file | :memory} | {:ttl, non_neg_integer()} @spec create_bucket(Gnat.t(), String.t(), list(bucket_opt)) :: {:ok, Stream.info()} | {:error, any()} def create_bucket(conn, bucket_name, params \\ []) do with :ok <- validate_bucket_name(bucket_name) do stream = %Stream{ name: stream_name(bucket_name), subjects: stream_subjects(bucket_name), description: Keyword.get(params, :description), discard: :new, allow_rollup_hdrs: true, max_age: Keyword.get(params, :ttl, 0), max_bytes: Keyword.get(params, :max_bucket_size, -1), max_msg_size: Keyword.get(params, :max_chunk_size, -1), num_replicas: Keyword.get(params, :replicas, 1), storage: Keyword.get(params, :storage, :file), placement: Keyword.get(params, :placement), duplicate_window: adjust_duplicate_window(Keyword.get(params, :ttl, 0)) } Stream.create(conn, stream) end end @spec delete_bucket(Gnat.t(), String.t()) :: :ok | {:error, any} def delete_bucket(conn, bucket_name) do Stream.delete(conn, stream_name(bucket_name)) end @spec delete(Gnat.t(), String.t(), String.t()) :: :ok | {:error, any} def delete(conn, bucket_name, object_name) do with {:ok, meta = %Meta{}} <- info(conn, bucket_name, object_name), meta <- %Meta{meta | deleted: true}, topic <- meta_stream_topic(bucket_name, object_name), {:ok, body} <- Jason.encode(meta), {:ok, _msg} <- Gnat.request(conn, topic, body, headers: [{"Nats-Rollup", "sub"}]) do filter = chunk_stream_topic(meta) Stream.purge(conn, stream_name(bucket_name), nil, %{filter: filter}) end end @spec get(Gnat.t(), String.t(), String.t(), (binary -> any())) :: :ok | {:error, any} def get(conn, bucket_name, object_name, chunk_fun) do with {:ok, %{config: _stream}} <- Stream.info(conn, stream_name(bucket_name)), {:ok, meta} <- info(conn, bucket_name, object_name) do receive_chunks(conn, meta, chunk_fun) end end @spec info(Gnat.t(), String.t(), String.t()) :: {:ok, Meta.t()} | {:error, any} def info(conn, bucket_name, object_name) do with {:ok, _stream_info} <- Stream.info(conn, stream_name(bucket_name)) do Stream.get_message(conn, stream_name(bucket_name), %{ last_by_subj: meta_stream_topic(bucket_name, object_name) }) |> case do {:ok, message} -> meta = json_to_meta(message.data) {:ok, meta} error -> error end end end @type list_option :: {:show_deleted, boolean()} @spec list(Gnat.t(), String.t(), list(list_option())) :: {:error, any} | {:ok, list(Meta.t())} def list(conn, bucket_name, options \\ []) do with {:ok, %{config: stream}} <- Stream.info(conn, stream_name(bucket_name)), topic <- Util.reply_inbox(), {:ok, sub} <- Gnat.sub(conn, self(), topic), {:ok, consumer} <- Consumer.create(conn, %Consumer{ stream_name: stream.name, deliver_subject: topic, deliver_policy: :last_per_subject, filter_subject: meta_stream_subject(bucket_name), ack_policy: :none, max_ack_pending: nil, replay_policy: :instant, max_deliver: 1 }), {:ok, messages} <- receive_all_metas(sub, consumer.num_pending) do :ok = Gnat.unsub(conn, sub) :ok = Consumer.delete(conn, stream.name, consumer.name) show_deleted = Keyword.get(options, :show_deleted, false) if show_deleted do {:ok, messages} else {:ok, Enum.reject(messages, &(&1.deleted == true))} end end end @spec put(Gnat.t(), String.t(), String.t(), File.io_device()) :: {:ok, Meta.t()} | {:error, any()} def put(conn, bucket_name, object_name, io) do nuid = Util.nuid() chunk_topic = chunk_stream_topic(bucket_name, nuid) with {:ok, %{config: _}} <- Stream.info(conn, stream_name(bucket_name)), :ok <- purge_prior_chunks(conn, bucket_name, object_name), {:ok, chunks, size, digest} <- send_chunks(conn, io, chunk_topic) do object_meta = %Meta{ name: object_name, bucket: bucket_name, nuid: nuid, size: size, chunks: chunks, digest: "SHA-256=#{Base.url_encode64(digest)}" } topic = meta_stream_topic(bucket_name, object_name) body = Jason.encode!(object_meta) case Gnat.request(conn, topic, body, headers: [{"Nats-Rollup", "sub"}]) do {:ok, _} -> {:ok, object_meta} error -> error end end end @doc """ Returns true if the provided stream is an Object bucket, false otherwise ## Parameters * `stream_name` - the stream name to test """ @spec is_object_bucket_stream?(stream_name :: binary()) :: boolean() def is_object_bucket_stream?(stream_name) do String.starts_with?(stream_name, "OBJ_") end @doc """ Returns a list of all Object buckets """ @spec list_buckets(conn :: Gnat.t()) :: {:error, term()} | {:ok, list(String.t())} def list_buckets(conn) do with {:ok, %{streams: streams}} <- Stream.list(conn) do stream_names = streams |> Enum.flat_map(fn bucket -> if is_object_bucket_stream?(bucket) do [bucket |> String.trim_leading(@stream_prefix)] else [] end end) {:ok, stream_names} else {:error, reason} -> {:error, reason} end end defp stream_name(bucket_name) do "#{@stream_prefix}#{bucket_name}" end defp stream_subjects(bucket_name) do [ chunk_stream_subject(bucket_name), meta_stream_subject(bucket_name) ] end defp chunk_stream_subject(bucket_name) do "#{@subject_prefix}#{bucket_name}.C.>" end defp chunk_stream_topic(bucket_name, nuid) do "#{@subject_prefix}#{bucket_name}.C.#{nuid}" end defp chunk_stream_topic(%Meta{bucket: bucket, nuid: nuid}) do "#{@subject_prefix}#{bucket}.C.#{nuid}" end defp meta_stream_subject(bucket_name) do "#{@subject_prefix}#{bucket_name}.M.>" end defp meta_stream_topic(bucket_name, object_name) do key = Base.url_encode64(object_name) "#{@subject_prefix}#{bucket_name}.M.#{key}" end @two_minutes_in_nanoseconds 120_000_000_000 # The `duplicate_window` can't be greater than the `max_age`. The default `duplicate_window` # is 2 minutes. We'll keep the 2 minute window UNLESS the ttl is less than 2 minutes defp adjust_duplicate_window(ttl) when ttl > 0 and ttl < @two_minutes_in_nanoseconds, do: ttl defp adjust_duplicate_window(_ttl), do: @two_minutes_in_nanoseconds defp json_to_meta(json) do raw = Jason.decode!(json) %{ "bucket" => bucket, "chunks" => chunks, "digest" => digest, "name" => name, "nuid" => nuid, "size" => size } = raw %Meta{ bucket: bucket, chunks: chunks, digest: digest, deleted: Map.get(raw, "deleted", false), name: name, nuid: nuid, size: size } end defp purge_prior_chunks(conn, bucket, name) do case info(conn, bucket, name) do {:ok, meta} -> Stream.purge(conn, stream_name(bucket), nil, %{filter: chunk_stream_topic(meta)}) {:error, %{"code" => 404}} -> :ok {:error, other} -> {:error, other} end end defp receive_all_metas(sid, num_pending, messages \\ []) defp receive_all_metas(_sid, 0, messages) do {:ok, messages} end defp receive_all_metas(sid, remaining, messages) do receive do {:msg, %{sid: ^sid, body: body}} -> meta = json_to_meta(body) receive_all_metas(sid, remaining - 1, [meta | messages]) after 10_000 -> {:error, :timeout_waiting_for_messages} end end defp receive_chunks(conn, %Meta{} = meta, chunk_fun) do topic = chunk_stream_topic(meta) stream = stream_name(meta.bucket) inbox = Util.reply_inbox() {:ok, sub} = Gnat.sub(conn, self(), inbox) {:ok, consumer} = Consumer.create(conn, %Consumer{ stream_name: stream, deliver_subject: inbox, deliver_policy: :all, filter_subject: topic, ack_policy: :none, max_ack_pending: nil, replay_policy: :instant, max_deliver: 1, flow_control: true, idle_heartbeat: 5_000_000_000 }) :ok = receive_chunks(conn, sub, meta.chunks, chunk_fun) :ok = Gnat.unsub(conn, sub) :ok = Consumer.delete(conn, stream, consumer.name) end defp receive_chunks(_conn, _sub, 0, _chunk_fun) do :ok end defp receive_chunks(conn, sub, remaining, chunk_fun) do receive do # Flow control message with reply - respond to it {:msg, %{sid: ^sub, status: "100", description: "FlowControl Request", reply_to: reply}} when not is_nil(reply) -> Gnat.pub(conn, reply, "") receive_chunks(conn, sub, remaining, chunk_fun) # Flow control or heartbeat message without reply - get next message {:msg, %{sid: ^sub, body: "", status: "100"}} -> receive_chunks(conn, sub, remaining, chunk_fun) # Regular data message {:msg, %{sid: ^sub, body: body}} -> chunk_fun.(body) receive_chunks(conn, sub, remaining - 1, chunk_fun) after 10_000 -> {:error, :timeout_waiting_for_messages} end end @chunk_size 128 * 1024 defp send_chunks(conn, io, topic) do sha = :crypto.hash_init(:sha256) size = 0 chunks = 0 send_chunks(conn, io, topic, sha, size, chunks) end defp send_chunks(conn, io, topic, sha, size, chunks) do case IO.binread(io, @chunk_size) do :eof -> sha = :crypto.hash_final(sha) {:ok, chunks, size, sha} {:error, err} -> {:error, err} bytes -> sha = :crypto.hash_update(sha, bytes) size = size + byte_size(bytes) chunks = chunks + 1 case Gnat.request(conn, topic, bytes) do {:ok, _} -> send_chunks(conn, io, topic, sha, size, chunks) error -> error end end end defp validate_bucket_name(name) do case Regex.match?(~r/^[a-zA-Z0-9_-]+$/, name) do true -> :ok false -> {:error, "invalid bucket name"} end end end ================================================ FILE: lib/gnat/jetstream/api/stream.ex ================================================ defmodule Gnat.Jetstream.API.Stream do @moduledoc """ A module representing a NATS JetStream Stream. Learn more about Streams: https://docs.nats.io/nats-concepts/jetstream/streams ## The Jetstream.API.Stream struct The struct's mandatory fields are `:name` and `:subjects`. The rest will have the NATS default values set. Stream struct fields explanation: * `:allow_direct` - Allow higher performance, direct access to get individual messages. E.g. KeyValue * `:allow_msg_ttl` - Allow header initiated per-message TTLs. * `:allow_rollup_hdrs` - allows the use of the Nats-Rollup header to replace all contents of a stream, or subject in a stream, with a single new message. * `:deny_delete` - restricts the ability to delete messages from a stream via the API. Cannot be changed once set to true. * `:deny_purge` - restricts the ability to purge messages from a stream via the API. Cannot be change once set to true. * `:description` - a short description of the purpose of this stream. * `:discard` - determines what happens when a Stream reaches its limits. It has the following options: - `:old` - the default option. Old messages are deleted. - `:new` - refuses new messages. * `:discard_new_per_subject` - - allows to enable discarding new messages per subject when limits are reached. Requires `discard: :new` and the `:max_msgs_per_subject` to be configured. * `:domain` - JetStream domain, mainly used for leaf nodes. See [JetStream on Leaf Nodes](https://docs.nats.io/running-a-nats-service/configuration/leafnodes/jetstream_leafnodes). * `:duplicate_window` - the window within which to track duplicate messages, expressed in nanoseconds. * `:max_age` - maximum age of any message in the Stream, expressed in nanoseconds. * `:max_bytes` - how many bytes the Stream may contain. Adheres to `:discard`, removing oldest or refusing new messages if the Stream exceeds this size. * `:max_consumers` - how many Consumers can be defined for a given Stream, -1 for unlimited. * `:max_msg_size` - the largest message that will be accepted by the Stream. * `:max_msgs_per_subject` - For wildcard streams ensure that for every unique subject this many messages are kept - a per subject retention limit. Only available on nats-server versions greater than 2.3.0 * `:max_msgs` - how many messages may be in a Stream. Adheres to `:discard`, removing oldest or refusing new messages if the Stream exceeds this number of messages * `:mirror` - maintains a 1:1 mirror of another stream with name matching this property. When a mirror is configured subjects and sources must be empty. * `:mirror_direct` - Allow higher performance and unified direct access for mirrors as well. * `:name` - a name for the Stream. See [naming](https://docs.nats.io/running-a-nats-service/nats_admin/jetstream_admin/naming). * `:no_ack` - disables acknowledging messages that are received by the Stream. * `:num_replicas` - how many replicas to keep for each message. * `:placement` - placement directives to consider when placing replicas of this stream, random placement when unset. It has the following properties: - `:cluster` - the desired cluster name to place the stream. - `:tags` - tags required on servers hosting this stream. * `:retention` - how messages are retained in the Stream. Once this is exceeded, old messages are removed. It has the following options: - `:limits` - the default policy. - `:interest` - `:workqueue` * `:sealed` - sealed streams do not allow messages to be deleted via limits or API, sealed streams can not be unsealed via configuration update. Can only be set on already created streams via the Update API. * `:sources` - list of stream names to replicate into this stream. * `:storage` - the type of storage backend. Available options: - `:file` - `:memory` * `:compression` - If file-based and a compression algorithm is specified, the stream data will be compressed on disk. Valid options are "none" for no compression or "s2" for Snappy compression. * `:subjects` - a list of subjects to consume, supports wildcards. * `:subject_delete_marker_ttl` - Enables and sets a duration expressed in nanoseconds. for adding server markers for delete, purge and max age limits. * `:template_owner` - when the Stream is managed by a Stream Template this identifies the template that manages the Stream. """ import Gnat.Jetstream.API.Util @enforce_keys [:name, :subjects] @derive {Jason.Encoder, except: [:domain]} defstruct [ :description, :mirror, :name, :domain, :no_ack, :placement, :sources, :subjects, :template_owner, allow_direct: false, allow_msg_ttl: false, allow_rollup_hdrs: false, deny_delete: false, deny_purge: false, discard: :old, duplicate_window: 120_000_000_000, max_age: 0, max_bytes: -1, max_consumers: -1, max_msg_size: -1, max_msgs_per_subject: -1, max_msgs: -1, mirror_direct: false, num_replicas: 1, retention: :limits, sealed: false, storage: :file, subject_delete_marker_ttl: 0, discard_new_per_subject: false, compression: "none" ] @type nanoseconds :: non_neg_integer() @type placement :: %{ :cluster => binary(), optional(:tags) => list(binary()) } @type t :: %__MODULE__{ allow_direct: boolean(), allow_msg_ttl: boolean(), allow_rollup_hdrs: boolean(), deny_delete: boolean(), deny_purge: boolean(), description: nil | binary(), discard: :old | :new, domain: nil | binary(), duplicate_window: nil | nanoseconds(), max_age: nanoseconds(), max_bytes: integer(), max_consumers: integer(), max_msg_size: nil | integer(), max_msgs: integer(), max_msgs_per_subject: integer(), mirror: nil | source(), mirror_direct: boolean(), name: binary(), no_ack: nil | boolean(), num_replicas: pos_integer(), placement: nil | placement(), retention: :limits | :workqueue | :interest, sealed: boolean(), sources: nil | list(source()), storage: :file | :memory, subjects: nil | list(binary()), subject_delete_marker_ttl: nanoseconds(), template_owner: nil | binary(), discard_new_per_subject: boolean(), compression: binary() } @typedoc """ Stream source fields explained: * `:name` - stream name. * `:opt_start_seq` - sequence to start replicating from. * `:opt_start_time` - timestamp to start replicating from. * `:filter_subject` - replicate only a subset of messages based on filter. * `:external` - configuration referencing a stream source in another account or JetStream domain. It has the following parameters: - `:api` - the subject prefix that imports other account/domain `$JS.API.CONSUMER.>` subjects - `:deliver` - the delivery subject to use for push consumer """ @type source :: %{ :name => binary(), optional(:opt_start_seq) => integer(), optional(:opt_start_time) => DateTime.t(), optional(:filter_subject) => binary(), optional(:external) => %{ api: binary(), deliver: binary() } } @type info :: %{ cluster: nil | %{ optional(:name) => binary(), optional(:leader) => binary(), optional(:replicas) => list(%{ :active => nanoseconds(), :name => binary(), :current => boolean(), optional(:offline) => boolean(), optional(:lag) => non_neg_integer() }) }, config: t(), created: DateTime.t(), mirror: nil | source_info(), sources: nil | list(source_info()), state: state() } @type state :: %{ bytes: non_neg_integer(), consumer_count: non_neg_integer(), deleted: nil | [non_neg_integer()], first_seq: non_neg_integer(), first_ts: DateTime.t(), last_seq: non_neg_integer(), last_ts: DateTime.t(), lost: nil | list(%{msgs: [non_neg_integer()], bytes: non_neg_integer()}), messages: non_neg_integer(), num_deleted: nil | integer(), num_subjects: nil | integer(), subjects: nil | %{} | %{ binary() => non_neg_integer() } } @typedoc """ * `code` - HTTP like error code in the 300 to 500 range * `description` - A human friendly description of the error * `err_code` - The NATS error code unique to each kind of error """ @type response_error :: %{ :code => non_neg_integer(), optional(:description) => binary(), optional(:err_code) => non_neg_integer() } @type source_info :: %{ :active => nanoseconds(), :lag => non_neg_integer(), :name => binary(), optional(:external) => %{ api: binary(), deliver: binary() }, optional(:error) => response_error() } @type streams :: %{ limit: non_neg_integer(), offset: non_neg_integer(), streams: list(binary()), total: non_neg_integer() } @typedoc """ * `seq` - Stream sequence number of the message to retrieve, cannot be combined with `last_by_subj` * `last_by_subj` - Retrieves the last message for a given subject, cannot be combined with `seq` """ @type message_access_method :: %{ optional(:seq) => non_neg_integer(), optional(:last_by_subj) => binary() } @typedoc """ * `data` - The decoded message payload * `subject` - The subject the message was originally received on * `time` - The time the message was received * `seq` - The sequence number of the message in the Stream * `hdrs` - The decoded headers for the message """ @type message_response :: %{ :data => any(), :seq => non_neg_integer(), :subject => binary(), :time => DateTime.t(), :hdrs => nil | binary() } # @doc """ # Initialize a Stream struct # ## Examples # iex> %Stream{} = Gnat.Jetstream.API.Stream.new(:gnat, name: "NEW_STREAM", subjects: ["NEW_STREAM.subjects"]) # """ # @spec new(conn :: Gnat.t(), fields :: keyword()) :: t() # def new(conn, fields \\ []) do # %__MODULE__{} # end @doc """ Creates a new Stream. ## Examples iex> {:ok, %{created: _}} = Gnat.Jetstream.API.Stream.create(:gnat, %Gnat.Jetstream.API.Stream{name: "anewstream", subjects: ["anewsubject"]}) """ @spec create(conn :: Gnat.t(), stream :: t()) :: {:ok, info()} | {:error, any()} def create(conn, %__MODULE__{} = stream) do with :ok <- validate(stream), {:ok, stream} <- request( conn, "#{js_api(stream.domain)}.STREAM.CREATE.#{stream.name}", Jason.encode!(stream) ) do {:ok, to_info(stream)} end end @doc """ Updates a Stream. ## Examples iex> {:ok, %{created: _}} = Gnat.Jetstream.API.Stream.create(:gnat, %Gnat.Jetstream.API.Stream{name: "update_test_stream", subjects: ["update_subject"]}) iex> {:ok, _} = Gnat.Jetstream.API.Stream.update(:gnat, %Gnat.Jetstream.API.Stream{name: "update_test_stream", subjects: ["update_subject", "new.update_subject"]}) """ @spec update(conn :: Gnat.t(), stream :: t()) :: {:ok, info()} | {:error, any()} def update(conn, %__MODULE__{} = stream) do with :ok <- validate(stream), {:ok, stream} <- request( conn, "#{js_api(stream.domain)}.STREAM.UPDATE.#{stream.name}", Jason.encode!(stream) ) do {:ok, to_info(stream)} end end @doc """ Deletes a Stream and all its data. ## Examples iex> Gnat.Jetstream.API.Stream.create(:gnat, %Gnat.Jetstream.API.Stream{name: "delstream", subjects: ["delsubject"]}) iex> Gnat.Jetstream.API.Stream.delete(:gnat, "delstream") :ok iex> {:error, %{"code" => 404, "description" => "stream not found"}} = Gnat.Jetstream.API.Stream.delete(:gnat, "wrong_stream") """ @spec delete(conn :: Gnat.t(), stream_name :: binary(), domain :: nil | binary()) :: :ok | {:error, any()} def delete(conn, stream_name, domain \\ nil) when is_binary(stream_name) do with {:ok, _response} <- request(conn, "#{js_api(domain)}.STREAM.DELETE.#{stream_name}", "") do :ok end end @doc """ Purges all of data in the stream but doesn't delete the stream. ## Examples iex> Gnat.Jetstream.API.Stream.create(:gnat, %Gnat.Jetstream.API.Stream{name: "purgestream", subjects: ["purgesubject"]}) iex> Gnat.Jetstream.API.Stream.purge(:gnat, "purgestream") :ok iex> {:error, %{"code" => 404, "description" => "stream not found"}} = Gnat.Jetstream.API.Stream.purge(:gnat, "wrong_stream") """ @spec purge(conn :: Gnat.t(), stream_name :: binary(), domain :: nil | binary()) :: :ok | {:error, any()} def purge(conn, stream_name, domain \\ nil) when is_binary(stream_name) do with {:ok, _response} <- request(conn, "#{js_api(domain)}.STREAM.PURGE.#{stream_name}", "") do :ok end end @doc """ Purges some of the messages in a stream according to the supplied filter ## Examples iex> Gnat.Jetstream.API.Stream.create(:gnat, %Gnat.Jetstream.API.Stream{name: "pstream", subjects: ["psub1", "psub2"]}) iex> Gnat.Jetstream.API.Stream.purge(:gnat, "pstream", nil, %{filter: "psub1"}) :ok """ @type method :: %{filter: String.t()} @spec purge(conn :: Gnat.t(), stream_name :: binary(), domain :: nil | binary(), method) :: :ok | {:error, any()} def purge(conn, stream_name, domain, method) when is_binary(stream_name) do with :ok <- validate_purge_method(method), body <- Jason.encode!(method), {:ok, _response} <- request(conn, "#{js_api(domain)}.STREAM.PURGE.#{stream_name}", body) do :ok end end @doc """ Information about config and state of a Stream. ## Examples iex> {:ok, _} = Gnat.Jetstream.API.Stream.create(:gnat, %Gnat.Jetstream.API.Stream{name: "infostream", subjects: ["infosubject"]}) iex> {:ok, %{created: _}} = Gnat.Jetstream.API.Stream.info(:gnat, "infostream") iex> {:error, %{"code" => 404, "description" => "stream not found"}} = Gnat.Jetstream.API.Stream.info(:gnat, "wrong_stream") """ @spec info(conn :: Gnat.t(), stream_name :: binary(), domain :: nil | binary()) :: {:ok, info()} | {:error, any()} def info(conn, stream_name, domain \\ nil) when is_binary(stream_name) do with {:ok, decoded} <- request(conn, "#{js_api(domain)}.STREAM.INFO.#{stream_name}", "") do {:ok, to_info(decoded)} end end @doc """ Paged list of known Streams including all their current information. ## Options * `:offset` - Number of records to skip * `:subject` - A subject the `Stream` must collect to appear in the list. ## Examples iex> {:ok, %{total: _, offset: 0, limit: 1024, streams: _}} = Gnat.Jetstream.API.Stream.list(:gnat) """ @spec list( conn :: Gnat.t(), params :: [offset: non_neg_integer(), subject: binary(), domain: nil | binary()] ) :: {:ok, streams()} | {:error, term()} def list(conn, params \\ []) do domain = Keyword.get(params, :domain) payload = Jason.encode!(%{ offset: Keyword.get(params, :offset, 0), subject: Keyword.get(params, :subject) }) with {:ok, decoded} <- request(conn, "#{js_api(domain)}.STREAM.NAMES", payload) do # Recent versions of NATS sometimes return `"streams": null` in their JSON payload to indicate # that no streams are defined. But, that would mean callers have to handle both `nil` and a list, so # we coerce that to an empty list to represent no streams being defined. streams = case Map.get(decoded, "streams") do nil -> [] names when is_list(names) -> names end result = %{ limit: Map.get(decoded, "limit"), offset: Map.get(decoded, "offset"), streams: streams, total: Map.get(decoded, "total") } {:ok, result} end end @doc """ Get a message from the stream either by "stream sequence number" or the "last message for a given subject" """ @spec get_message( conn :: Gnat.t(), stream_name :: binary(), method :: message_access_method(), domain :: nil | binary() ) :: {:ok, message_response()} | {:error, response_error()} def get_message(conn, stream_name, method, domain \\ nil) when is_map(method) do with :ok <- validate_message_access_method(method), {:ok, %{"message" => message}} <- request(conn, "#{js_api(domain)}.STREAM.MSG.GET.#{stream_name}", Jason.encode!(method)) do {:ok, %{ data: decode_base64(message["data"]), seq: message["seq"], subject: message["subject"], time: to_datetime(message["time"]), hdrs: decode_base64(message["hdrs"]) }} end end # https://docs.nats.io/running-a-nats-service/configuration/leafnodes/jetstream_leafnodes defp js_api(nil), do: "$JS.API" defp js_api(""), do: "$JS.API" defp js_api(domain), do: "$JS.#{domain}.API" defp to_state(state) do %{ bytes: Map.fetch!(state, "bytes"), consumer_count: Map.fetch!(state, "consumer_count"), deleted: Map.get(state, "deleted"), first_seq: Map.fetch!(state, "first_seq"), first_ts: Map.fetch!(state, "first_ts") |> to_datetime(), last_seq: Map.fetch!(state, "last_seq"), last_ts: Map.fetch!(state, "last_ts") |> to_datetime(), lost: Map.get(state, "lost"), messages: Map.fetch!(state, "messages"), num_deleted: Map.get(state, "num_deleted"), num_subjects: Map.get(state, "num_subjects"), subjects: Map.get(state, "subjects") } end defp to_stream(stream) do %__MODULE__{ description: Map.get(stream, "description"), discard: Map.fetch!(stream, "discard") |> to_sym(), duplicate_window: Map.get(stream, "duplicate_window"), max_age: Map.fetch!(stream, "max_age"), max_bytes: Map.fetch!(stream, "max_bytes"), max_consumers: Map.fetch!(stream, "max_consumers"), max_msg_size: Map.get(stream, "max_msg_size"), max_msgs_per_subject: Map.get(stream, "max_msgs_per_subject", -1), max_msgs: Map.fetch!(stream, "max_msgs"), mirror: Map.get(stream, "mirror"), name: Map.fetch!(stream, "name"), no_ack: Map.get(stream, "no_ack"), num_replicas: Map.fetch!(stream, "num_replicas"), placement: Map.get(stream, "placement"), retention: Map.fetch!(stream, "retention") |> to_sym(), sources: Map.get(stream, "sources"), storage: Map.fetch!(stream, "storage") |> to_sym(), subjects: Map.get(stream, "subjects"), template_owner: Map.get(stream, "template_owner") } # Check for fields added in NATS versions higher than 2.2.0 |> put_if_exist(:allow_direct, stream, "allow_direct") |> put_if_exist(:allow_msg_ttl, stream, "allow_msg_ttl") |> put_if_exist(:allow_rollup_hdrs, stream, "allow_rollup_hdrs") |> put_if_exist(:deny_delete, stream, "deny_delete") |> put_if_exist(:deny_purge, stream, "deny_purge") |> put_if_exist(:discard_new_per_subject, stream, "discard_new_per_subject") |> put_if_exist(:mirror_direct, stream, "mirror_direct") |> put_if_exist(:sealed, stream, "sealed") |> put_if_exist(:subject_delete_marker_ttl, stream, "subject_delete_marker_ttl") |> put_if_exist(:compression, stream, "compression") end defp to_info(%{"config" => config, "state" => state, "created" => created} = response) do with {:ok, created, _} <- DateTime.from_iso8601(created) do %{ cluster: Map.get(response, "cluster"), config: to_stream(config), created: created, mirror: Map.get(response, "mirror"), sources: Map.get(response, "sources"), state: to_state(state) } end end defp validate(stream_settings) do cond do Map.has_key?(stream_settings, :name) == false -> {:error, "Must have a :name set"} is_binary(Map.get(stream_settings, :name)) == false -> {:error, "name must be a string"} valid_name?(stream_settings.name) == false -> {:error, "invalid name: " <> invalid_name_message()} Map.has_key?(stream_settings, :subjects) == false -> {:error, "You must specify a :subjects key"} is_list(Map.get(stream_settings, :subjects)) == false -> {:error, ":subjects must be a list of strings"} Enum.all?(Map.get(stream_settings, :subjects), &is_binary/1) == false -> {:error, ":subjects must be a list of strings"} true -> :ok end end defp validate_message_access_method(method) do if map_size(method) == 1 do :ok else {:error, "To get a message you must use only one of `seq` or `last_by_subj`"} end end defp validate_purge_method(%{filter: subject}) when is_binary(subject) do :ok end defp validate_purge_method(_) do {:error, "When purging, you must pass a %{filter: subject}"} end end ================================================ FILE: lib/gnat/jetstream/api/util.ex ================================================ defmodule Gnat.Jetstream.API.Util do @moduledoc false @default_inbox_prefix "_INBOX." def request(conn, topic, payload) do with {:ok, %{body: body}} <- Gnat.request(conn, topic, payload), {:ok, decoded} <- Jason.decode(body) do case decoded do %{"error" => err} -> {:error, err} other -> {:ok, other} end end end def to_datetime(nil), do: nil def to_datetime(str) do {:ok, datetime, _} = DateTime.from_iso8601(str) datetime end def to_sym(nil), do: nil def to_sym(str) when is_binary(str) do String.to_existing_atom(str) end def put_if_exist(target_map, target_key, source_map, source_key) do case Map.fetch(source_map, source_key) do {:ok, value} -> Map.put(target_map, target_key, value) _ -> target_map end end def valid_name?(name) do !String.contains?(name, [".", "*", ">", " ", "\t"]) end def invalid_name_message do "cannot contain '.', '>', '*', spaces or tabs" end def decode_base64(nil), do: nil def decode_base64(data), do: Base.decode64!(data) def reply_inbox(prefix \\ @default_inbox_prefix) def reply_inbox(nil), do: reply_inbox() def reply_inbox(prefix), do: prefix <> nuid() def nuid() do :crypto.strong_rand_bytes(12) |> Base.url_encode64() end end ================================================ FILE: lib/gnat/jetstream/jetstream.ex ================================================ defmodule Gnat.Jetstream do @moduledoc """ Provides functions for interacting with a [NATS Jetstream](https://github.com/nats-io/jetstream) server. """ @doc """ Sends `AckAck` acknowledgement to the server. Acknowledges a message was completely handled. """ @spec ack(message :: Gnat.message()) :: :ok def ack(%{reply_to: nil}) do {:error, "Cannot ack message with no reply-to"} end def ack(%{gnat: gnat, reply_to: reply_to}) do Gnat.pub(gnat, reply_to, "") end @doc """ Sends `AckNext` acknowledgement to the server. Acknowledges the message was handled and requests delivery of the next message to the reply subject. Only applies to Pull-mode. """ @spec ack_next(message :: Gnat.message(), consumer_subject :: binary()) :: :ok def ack_next(%{reply_to: nil}, _consumer_subject) do {:error, "Cannot ack message with no reply-to"} end def ack_next(%{gnat: gnat, reply_to: reply_to}, consumer_subject) do Gnat.pub(gnat, reply_to, "+NXT", reply_to: consumer_subject) end @doc """ Sends `AckNak` acknowledgement to the server. Signals that the message will not be processed now and processing can move onto the next message. NAK'd message will be retried. """ @spec nack(message :: Gnat.message()) :: :ok def nack(%{reply_to: nil}) do {:error, "Cannot ack message with no reply-to"} end def nack(%{gnat: gnat, reply_to: reply_to}) do Gnat.pub(gnat, reply_to, "-NAK") end @doc """ Sends `AckProgress` acknowledgement to the server. When sent before the `AckWait` period indicates that work is ongoing and the period should be extended by another equal to `AckWait`. """ @spec ack_progress(message :: Gnat.message()) :: :ok def ack_progress(%{reply_to: nil}) do {:error, "Cannot ack message with no reply-to"} end def ack_progress(%{gnat: gnat, reply_to: reply_to}) do Gnat.pub(gnat, reply_to, "+WPI") end @doc """ Sends `AckTerm` acknowledgement to the server. Instructs the server to stop redelivery of a message without acknowledging it as successfully processed. """ @spec ack_term(message :: Gnat.message()) :: :ok def ack_term(%{reply_to: nil}) do {:error, "Cannot ack message with no reply-to"} end def ack_term(%{gnat: gnat, reply_to: reply_to}) do Gnat.pub(gnat, reply_to, "+TERM") end end ================================================ FILE: lib/gnat/jetstream/pager.ex ================================================ defmodule Gnat.Jetstream.Pager do @moduledoc """ Page through all the messages in a stream This module provides a synchronous API to inspect the messages in a stream. You can use the reduce module to write a simple function that works like `Enum.reduce` across each message individually. If you want to handle messages in batches, you can use the `init` + `page` functions to accomplish that. """ alias Gnat.Jetstream alias Gnat.Jetstream.API.{Consumer, Util} @opaque pager :: map() @type message :: Gnat.message() @type opts :: list(opt()) @typedoc """ Options you can pass to the pager * `batch` controls the maximum number of messages we'll pull in each page/batch (default 10) * `domain` You can specify a jetstream domain if needed * `from_datetime` Only page through messages recorded on or after this datetime * `from_seq` Only page through messages with a sequence number equal or above this option * `headers_only` You can pass `true` to this if you only want to see the headers from each message. Can be useful to get metadata without having to receieve large body payloads. """ @type opt :: {:batch, non_neg_integer()} | {:domain, String.t()} | {:from_datetime, DateTime.t()} | {:from_seq, non_neg_integer} | {:headers_only, boolean()} @spec init(Gnat.t(), String.t(), opts()) :: {:ok, pager()} | {:error, term()} def init(conn, stream_name, opts) do domain = Keyword.get(opts, :domain) consumer = %Consumer{ stream_name: stream_name, domain: domain, ack_policy: :all, ack_wait: 30_000_000_000, deliver_policy: :all, description: "ephemeral consumer", replay_policy: :instant, inactive_threshold: 30_000_000_000, headers_only: Keyword.get(opts, :headers_only) } consumer = apply_opts_to_consumer(consumer, opts) inbox = Util.reply_inbox() with {:ok, consumer_info} <- Consumer.create(conn, consumer), {:ok, sub} <- Gnat.sub(conn, self(), inbox) do state = %{ conn: conn, stream_name: stream_name, consumer_name: consumer_info.name, domain: domain, inbox: inbox, batch: Keyword.get(opts, :batch, 10), sub: sub } {:ok, state} end end @spec page(pager()) :: {:page, list(message())} | {:done, list(message())} | {:error, term()} def page(state) do with :ok <- request_next_message(state) do receive_messages(state, []) end end def cleanup(%{conn: conn} = state) do :ok = Gnat.unsub(conn, state.sub) Consumer.delete(conn, state.stream_name, state.consumer_name, state.domain) end @doc """ Similar to Enum.reduce but you can iterate through all messages in a stream ``` # Assume we have a stream with messages like "1", "2", ... "10" Gnat.Jetstream.Pager.reduce(:gnat, "NUMBERS_STREAM", [batch_size: 5], 0, fn(message, total) -> num = String.to_integer(message.body) total + num end) # => {:ok, 55} ``` """ @spec reduce( Gnat.t(), String.t(), opts(), Enum.acc(), (Gnat.message(), Enum.acc() -> Enum.acc()) ) :: {:ok, Enum.acc()} | {:error, term()} def reduce(conn, stream_name, opts, initial_state, fun) do with {:ok, pager} <- init(conn, stream_name, opts) do page_through(pager, initial_state, fun) end end defp page_through(pager, state, fun) do case page(pager) do {:page, messages} -> new_state = Enum.reduce(messages, state, fun) page_through(pager, new_state, fun) {:done, messages} -> new_state = Enum.reduce(messages, state, fun) cleanup(pager) {:ok, new_state} end end defp request_next_message(state) do opts = [batch: state.batch, no_wait: true] Consumer.request_next_message( state.conn, state.stream_name, state.consumer_name, state.inbox, state.domain, opts ) end defp receive_messages(%{batch: batch}, messages) when length(messages) == batch do last = hd(messages) :ok = Jetstream.ack(last) {:page, Enum.reverse(messages)} end @terminals ["404", "408"] defp receive_messages(%{sub: sid} = state, messages) do receive do {:msg, %{sid: ^sid, status: status}} when status in @terminals -> {:done, Enum.reverse(messages)} {:msg, %{sid: ^sid, reply_to: nil}} -> {:done, Enum.reverse(messages)} {:msg, %{sid: ^sid} = message} -> receive_messages(state, [message | messages]) end end ## Helpers for accepting user options defp apply_opts_to_consumer(consumer = %Consumer{}, opts) do from = {Keyword.get(opts, :from_seq), Keyword.get(opts, :from_datetime)} case from do {nil, nil} -> consumer {seq, _} when is_integer(seq) -> %Consumer{consumer | deliver_policy: :by_start_sequence, opt_start_seq: seq} {_, %DateTime{} = dt} -> %Consumer{consumer | deliver_policy: :by_start_time, opt_start_time: dt} end end end ================================================ FILE: lib/gnat/jetstream/pull_consumer/connection_options.ex ================================================ defmodule Gnat.Jetstream.PullConsumer.ConnectionOptions do @moduledoc false @default_retry_timeout 1000 @default_retries 10 @enforce_keys [ :connection_name, :connection_retry_timeout, :connection_retries, :inbox_prefix, :domain ] # 5 Seconds in nanoseconds @default_request_expires 5_000_000_000 defstruct @enforce_keys ++ [ :stream_name, :consumer_name, :consumer, batch_size: 1, request_expires: @default_request_expires ] def validate!(connection_options) do validated_opts = Keyword.validate!(connection_options, [ :connection_name, :stream_name, :consumer_name, :consumer, connection_retry_timeout: @default_retry_timeout, connection_retries: @default_retries, inbox_prefix: nil, domain: nil, batch_size: 1, request_expires: @default_request_expires ]) stream_name = validated_opts[:stream_name] consumer_name = validated_opts[:consumer_name] consumer = validated_opts[:consumer] cond do consumer && (stream_name || consumer_name) -> raise ArgumentError, "cannot specify :consumer with :stream_name or :consumer_name - use consumer struct's stream_name instead" consumer && !is_struct(consumer, Gnat.Jetstream.API.Consumer) -> raise ArgumentError, ":consumer must be a Consumer struct" consumer && consumer.durable_name != nil && consumer.inactive_threshold == nil -> raise ArgumentError, "durable consumers specified via :consumer must have inactive_threshold set for auto-cleanup" consumer && validated_opts[:batch_size] > 1 && consumer.ack_policy != :all -> raise ArgumentError, "batch_size > 1 requires ack_policy: :all on the consumer, " <> "got: #{inspect(consumer.ack_policy)}. With ack_policy: :explicit, " <> "only the last message in each batch would be acknowledged and the " <> "server would redeliver the rest" consumer -> # For ephemeral/auto-cleanup consumer case, extract stream_name from consumer struct validated_opts = Keyword.put(validated_opts, :stream_name, consumer.stream_name) struct!(__MODULE__, validated_opts) stream_name && consumer_name -> # For traditional durable consumer case struct!(__MODULE__, validated_opts) true -> raise ArgumentError, "must specify either :consumer (ephemeral/auto-cleanup) or both :stream_name and :consumer_name (durable)" end end end ================================================ FILE: lib/gnat/jetstream/pull_consumer/server.ex ================================================ defmodule Gnat.Jetstream.PullConsumer.Server do @moduledoc false require Logger use Connection alias Gnat.Jetstream.PullConsumer.ConnectionOptions alias Gnat.Jetstream.API.Util defstruct [ :connection_options, :state, :listening_topic, :module, :subscription_id, :connection_monitor_ref, :consumer_name, current_retry: 0, buffer: [] ] def init(%{module: module, init_arg: init_arg}) do _ = Process.put(:"$initial_call", {module, :init, 1}) case module.init(init_arg) do {:ok, state, connection_options} when is_list(connection_options) -> Process.flag(:trap_exit, true) connection_options = ConnectionOptions.validate!(connection_options) gen_state = %__MODULE__{ connection_options: connection_options, state: state, listening_topic: Util.reply_inbox(connection_options.inbox_prefix), module: module, consumer_name: connection_options.consumer_name } {:connect, :init, gen_state} :ignore -> :ignore {:stop, _} = stop -> stop end end def connect( _, %__MODULE__{ connection_options: %ConnectionOptions{ stream_name: stream_name, consumer_name: consumer_name, consumer: consumer, connection_name: connection_name, connection_retry_timeout: connection_retry_timeout, connection_retries: connection_retries, domain: domain }, listening_topic: listening_topic, module: module } = gen_state ) do Logger.debug( "#{__MODULE__} for #{stream_name}.#{gen_state.consumer_name} is connecting to Gnat.", module: module, listening_topic: listening_topic, connection_name: connection_name ) with {:ok, conn} <- connection_pid(connection_name), monitor_ref = Process.monitor(conn), {:ok, consumer_info} <- ensure_consumer_exists( conn, stream_name, consumer_name, consumer, domain ), :ok <- validate_batch_ack_policy(gen_state.connection_options, consumer_info), final_consumer_name = consumer_info.name, state = maybe_handle_connected(module, consumer_info, gen_state.state), {:ok, sid} <- Gnat.sub(conn, self(), listening_topic), gen_state = %{ gen_state | subscription_id: sid, connection_monitor_ref: monitor_ref, consumer_name: final_consumer_name, state: state }, :ok <- initial_fetch( gen_state, conn, stream_name, final_consumer_name, domain, listening_topic ), gen_state = %{gen_state | current_retry: 0} do {:ok, gen_state} else {:error, reason} -> if gen_state.current_retry >= connection_retries do Logger.error( """ #{__MODULE__} for #{stream_name}.#{gen_state.consumer_name} failed to connect to NATS and \ retries limit has been exhausted. Stopping. """, module: module, listening_topic: listening_topic, connection_name: connection_name ) {:stop, :timeout, %{gen_state | current_retry: 0}} else Logger.debug( """ #{__MODULE__} for #{stream_name}.#{gen_state.consumer_name} failed to connect to Gnat \ and will retry. Reason: #{inspect(reason)} """, module: module, listening_topic: listening_topic, connection_name: connection_name ) gen_state = Map.update!(gen_state, :current_retry, &(&1 + 1)) {:backoff, connection_retry_timeout, gen_state} end end end def disconnect( {:close, from}, %__MODULE__{ connection_options: %ConnectionOptions{ stream_name: stream_name, connection_name: connection_name }, listening_topic: listening_topic, subscription_id: subscription_id, module: module, consumer_name: consumer_name } = gen_state ) do Logger.debug( "#{__MODULE__} for #{stream_name}.#{consumer_name} is disconnecting from Gnat.", module: module, listening_topic: listening_topic, subscription_id: subscription_id, connection_name: connection_name ) with {:ok, conn} <- connection_pid(connection_name), true <- Process.demonitor(gen_state.connection_monitor_ref, [:flush]), :ok <- Gnat.unsub(conn, subscription_id) do Logger.debug( "#{__MODULE__} for #{stream_name}.#{consumer_name} is shutting down.", module: module, listening_topic: listening_topic, subscription_id: subscription_id, connection_name: connection_name ) Connection.reply(from, :ok) {:stop, :shutdown, gen_state} end end defp ensure_consumer_exists(gnat, stream_name, consumer_name, nil, domain) do # Durable consumer case - just check it exists try do case Gnat.Jetstream.API.Consumer.info(gnat, stream_name, consumer_name, domain) do {:ok, consumer_info} -> {:ok, consumer_info} {:error, reason} -> {:error, reason} end catch :exit, reason -> {:error, {:process_exit, reason}} kind, reason -> {:error, {kind, reason}} end end defp ensure_consumer_exists(gnat, _stream_name, nil, consumer_struct, _domain) do # Ephemeral or auto-cleanup durable consumer case - create it try do with {:ok, consumer_definition} <- validate_consumer_for_creation(consumer_struct), {:ok, consumer_info} <- Gnat.Jetstream.API.Consumer.create(gnat, consumer_definition) do {:ok, consumer_info} end catch :exit, reason -> {:error, {:process_exit, reason}} kind, reason -> {:error, {kind, reason}} end end defp validate_consumer_for_creation(consumer_definition) do cond do consumer_definition.durable_name == nil && consumer_definition.inactive_threshold != nil -> {:error, "ephemeral consumers (durable_name: nil) cannot have inactive_threshold set"} consumer_definition.durable_name != nil && consumer_definition.inactive_threshold == nil -> {:error, "durable consumers specified via :consumer must have inactive_threshold set for auto-cleanup"} true -> {:ok, consumer_definition} end end defp validate_batch_ack_policy(%ConnectionOptions{batch_size: batch_size}, consumer_info) when batch_size > 1 do case consumer_info.config.ack_policy do :all -> :ok other -> {:error, "batch_size > 1 requires ack_policy: :all on the consumer, " <> "got: #{inspect(other)}. With ack_policy: :explicit, " <> "only the last message in each batch would be acknowledged and the " <> "server would redeliver the rest"} end end defp validate_batch_ack_policy(_connection_options, _consumer_info), do: :ok defp maybe_handle_connected(module, consumer_info, state) do if function_exported?(module, :handle_connected, 2) do {:ok, state} = module.handle_connected(consumer_info, state) state else state end end defp maybe_handle_status(message, %__MODULE__{module: module, state: state} = gen_state) do if function_exported?(module, :handle_status, 2) do {:ok, new_state} = module.handle_status(message, state) %{gen_state | state: new_state} else gen_state end end defp connection_pid(connection_name) when is_pid(connection_name) do if Process.alive?(connection_name) do {:ok, connection_name} else {:error, :not_alive} end end defp connection_pid(connection_name) do case Process.whereis(connection_name) do nil -> {:error, :not_found} pid -> {:ok, pid} end end # -- Batch mode: 100 is an idle heartbeat — the pull is still alive, do # nothing but invoke the user callback. -- def handle_info( {:msg, %{status: "100"} = message}, %__MODULE__{connection_options: %ConnectionOptions{batch_size: batch_size}} = gen_state ) when batch_size > 1 do gen_state = maybe_handle_status(message, gen_state) {:noreply, gen_state} end # -- Batch mode: any other status (404/408 terminators, 409 leadership # change / max_ack_pending / max_waiting / consumer-deleted, etc.) ends # the outstanding pull request. Process any partial buffer and issue a # new pull so the consumer doesn't stall. -- def handle_info( {:msg, %{status: status, gnat: gnat} = message}, %__MODULE__{ connection_options: %ConnectionOptions{batch_size: batch_size}, buffer: buffer } = gen_state ) when batch_size > 1 and is_binary(status) and status != "" do gen_state = maybe_handle_status(message, gen_state) case buffer do [] -> # Nothing buffered — long-poll for new messages. request_batch(gnat, gen_state, :tailing) {:noreply, gen_state} _messages -> # Partial batch — process what we have, then try for more. gen_state = process_and_ack_batch(gen_state) request_batch(gnat, gen_state, :catching_up) {:noreply, gen_state} end end # -- Single-message mode: informational status. Drop + re-pull so the # consumer doesn't stall. Matches the nats.go convention of never exposing # status messages to the user's message handler. -- def handle_info( {:msg, %{status: status} = message}, %__MODULE__{ connection_options: %ConnectionOptions{ stream_name: stream_name, domain: domain }, listening_topic: listening_topic, consumer_name: consumer_name } = gen_state ) when is_binary(status) and status != "" do gen_state = maybe_handle_status(message, gen_state) next_message( message.gnat, stream_name, consumer_name, domain, listening_topic ) {:noreply, gen_state} end # -- Batch mode: data message — buffer until batch is full -- def handle_info( {:msg, message}, %__MODULE__{ connection_options: %ConnectionOptions{batch_size: batch_size}, buffer: buffer } = gen_state ) when batch_size > 1 do buffer = [message | buffer] gen_state = %{gen_state | buffer: buffer} if length(buffer) >= batch_size do gen_state = process_and_ack_batch(gen_state) request_batch(message.gnat, gen_state, :catching_up) {:noreply, gen_state} else {:noreply, gen_state} end end # -- Single-message mode (batch_size == 1, the default) -- def handle_info( {:msg, message}, %__MODULE__{ connection_options: %ConnectionOptions{ stream_name: stream_name, connection_name: connection_name, domain: domain }, listening_topic: listening_topic, subscription_id: subscription_id, state: state, module: module, consumer_name: consumer_name } = gen_state ) do Logger.debug( """ #{__MODULE__} for #{stream_name}.#{consumer_name} received a message: \ #{inspect(message, pretty: true)} """, module: module, listening_topic: listening_topic, subscription_id: subscription_id, connection_name: connection_name ) case module.handle_message(message, state) do {:ack, state} -> Gnat.Jetstream.ack_next(message, listening_topic) gen_state = %{gen_state | state: state} {:noreply, gen_state} {:nack, state} -> Gnat.Jetstream.nack(message) next_message( message.gnat, stream_name, consumer_name, domain, listening_topic ) gen_state = %{gen_state | state: state} {:noreply, gen_state} {:term, state} -> Gnat.Jetstream.ack_term(message) next_message( message.gnat, stream_name, consumer_name, domain, listening_topic ) gen_state = %{gen_state | state: state} {:noreply, gen_state} {:noreply, state} -> next_message( message.gnat, stream_name, consumer_name, domain, listening_topic ) gen_state = %{gen_state | state: state} {:noreply, gen_state} end end def handle_info( {:DOWN, ref, :process, _pid, _reason}, %__MODULE__{ connection_options: %ConnectionOptions{ connection_name: connection_name, stream_name: stream_name }, subscription_id: subscription_id, listening_topic: listening_topic, module: module, connection_monitor_ref: monitor_ref, consumer_name: consumer_name } = gen_state ) when ref == monitor_ref do Logger.debug( """ #{__MODULE__} for #{stream_name}.#{consumer_name}: NATS connection has died. PullConsumer is reconnecting. """, module: module, listening_topic: listening_topic, subscription_id: subscription_id, connection_name: connection_name ) # Clear consumer name on reconnect so it gets recreated (for ephemeral and auto-cleanup consumers) # Clear buffer to avoid processing stale messages from the dead connection gen_state = %{ gen_state | consumer_name: nil, subscription_id: nil, connection_monitor_ref: nil, buffer: [] } {:connect, :reconnect, gen_state} end def handle_info( other, %__MODULE__{ connection_options: %ConnectionOptions{ connection_name: connection_name, stream_name: stream_name }, subscription_id: subscription_id, listening_topic: listening_topic, module: module, consumer_name: consumer_name } = gen_state ) do Logger.debug( """ #{__MODULE__} for #{stream_name}.#{consumer_name} received unexpected message: #{inspect(other, pretty: true)} """, module: module, listening_topic: listening_topic, subscription_id: subscription_id, connection_name: connection_name ) {:noreply, gen_state} end def handle_call( :close, from, %__MODULE__{ connection_options: %ConnectionOptions{ connection_name: connection_name, stream_name: stream_name }, subscription_id: subscription_id, listening_topic: listening_topic, module: module, consumer_name: consumer_name } = gen_state ) do Logger.debug("#{__MODULE__} for #{stream_name}.#{consumer_name} received :close call.", module: module, listening_topic: listening_topic, subscription_id: subscription_id, connection_name: connection_name ) {:disconnect, {:close, from}, gen_state} end defp next_message(conn, stream_name, consumer_name, domain, listening_topic) do Gnat.Jetstream.API.Consumer.request_next_message( conn, stream_name, consumer_name, listening_topic, domain ) end defp initial_fetch(gen_state, conn, stream_name, consumer_name, domain, listening_topic) do if gen_state.connection_options.batch_size > 1 do request_batch(conn, gen_state, :catching_up) else next_message(conn, stream_name, consumer_name, domain, listening_topic) end end defp request_batch(conn, gen_state, mode) do %{ connection_options: %ConnectionOptions{ stream_name: stream_name, batch_size: batch_size, domain: domain, request_expires: expires }, consumer_name: consumer_name, listening_topic: listening_topic } = gen_state opts = case mode do :catching_up -> [batch: batch_size, no_wait: true] :tailing -> [batch: batch_size, expires: expires] end Gnat.Jetstream.API.Consumer.request_next_message( conn, stream_name, consumer_name, listening_topic, domain, opts ) end defp process_and_ack_batch(%{buffer: buffer, module: module, state: state} = gen_state) do messages = Enum.reverse(buffer) new_state = Enum.reduce(messages, state, fn message, acc_state -> case module.handle_message(message, acc_state) do {:ack, updated_state} -> updated_state {action, updated_state} -> Logger.warning( "PullConsumer batch mode does not support #{inspect(action)}, treating as :ack" ) updated_state end end) # With ack_policy: :all, acking the last message covers the entire batch last = List.last(messages) Gnat.Jetstream.ack(last) %{gen_state | state: new_state, buffer: []} end end ================================================ FILE: lib/gnat/jetstream/pull_consumer.ex ================================================ defmodule Gnat.Jetstream.PullConsumer do @moduledoc """ A behaviour which provides the NATS JetStream Pull Consumer functionalities. When a Consumer is pull-based, it means that the messages will be delivered when the server is asked for them. ## Example Declare a module which uses `Gnat.Jetstream.PullConsumer` and implements `c:init/1` and `c:handle_message/2` callbacks. defmodule MyApp.PullConsumer do use Gnat.Jetstream.PullConsumer def start_link(arg) do Jetstream.PullConsumer.start_link(__MODULE__, arg) end @impl true def init(_arg) do {:ok, nil, connection_name: :gnat, stream_name: "TEST_STREAM", consumer_name: "TEST_CONSUMER"} end @impl true def handle_message(message, state) do # Do some processing with the message. {:ack, state} end end You can then place your Pull Consumer in a supervision tree. Remember that you need to have the `Gnat.ConnectionSupervisor` set up. defmodule MyApp.Application do use Application @impl true def start(_type, _args) do children = [ # Create NATS connection {Gnat.ConnectionSupervisor, ...}, # Start NATS Jetstream Pull Consumer MyApp.PullConsumer, ] opts = [strategy: :one_for_one] Supervisor.start_link(children, opts) end end ## Connection Options In order to establish consumer connection with NATS, you need to pass several connection options via keyword list in third element of a tuple returned from `c:init/1` callback. Following options **must** be provided. Omitting this options will cause the process to raise errors upon initialization: * `:connection_name` - Gnat connection or `Gnat.ConnectionSupervisor` name/PID. For **durable consumers**, provide: * `:stream_name` - name of an existing stream the consumer will consume messages from. * `:consumer_name` - name of an existing consumer pointing at the stream. For **ephemeral consumers**, provide: * `:consumer` - a `Gnat.Jetstream.API.Consumer` struct for creating an ephemeral consumer. The consumer struct must have `durable_name: nil` OR `inactive_threshold` set to ensure that the server will clean it up. The `stream_name` field must also be set. You can also pass the optional ones: * `:connection_retry_timeout` - a duration in milliseconds after which the PullConsumer which failed to establish NATS connection retries, defaults to `1000` * `:connection_retries` - a number of attempts the PullConsumer will make to establish the NATS connection. When this value is exceeded, the pull consumer stops with the `:timeout` reason, defaults to `10` * `:inbox_prefix` - allows the default `_INBOX.` prefix to be customized. Should end with a dot. * `:domain` - use a JetStream domain, this is mostly used on leaf nodes. * `:batch_size` - when set to a value greater than 1, enables batch mode. Messages are buffered and delivered to `c:handle_message/2` in batches. Only the last message per batch is acknowledged, so the underlying consumer should use `ack_policy: :all` for correctness. This dramatically improves throughput for consumers that need to catch up on large backlogs. In batch mode, `:nack` and `:term` returns from `c:handle_message/2` are treated as `:ack` since `ack_policy: :all` cannot selectively reject messages. Defaults to `1` (single-message mode). * `:request_expires` - duration in nanoseconds a batch-mode pull request will linger on the server while tailing (no backlog) before the server replies with a `408` terminator and the consumer issues a fresh pull. Only used when `:batch_size` is greater than 1. Defaults to `5_000_000_000` (5 seconds). ## Dynamic Connection Options It is possible that you have to determine some of the options dynamically depending on pull consumer's init argument. To do so, it is recommended to derive these options values from some init argument: defmodule MyApp.PullConsumer do use Gnat.Jetstream.PullConsumer def start_link() do Gnat.Jetstream.PullConsumer.start_link(__MODULE__, %{counter: counter}) end @impl true def init(%{counter: counter}) do {:ok, nil, connection_name: :gnat, stream_name: "TEST_STREAM_#\{counter}", consumer_name: "TEST_CONSUMER_#\{counter}"} end ... end ## Ephemeral Consumer Example You can create ephemeral consumers by providing a `:consumer` struct with `durable_name: nil`. These are automatically created and cleaned up with the connection lifecycle: defmodule MyApp.EphemeralPullConsumer do use Gnat.Jetstream.PullConsumer def start_link(arg) do Jetstream.PullConsumer.start_link(__MODULE__, arg) end @impl true def init(_arg) do consumer = %Gnat.Jetstream.API.Consumer{ stream_name: "TEST_STREAM", durable_name: nil, # Must be nil for ephemeral consumers filter_subject: "orders.*" } {:ok, nil, connection_name: :gnat, consumer: consumer} end @impl true def handle_message(message, state) do # Do some processing with the message. {:ack, state} end end ## Auto-Cleanup Durable Consumer Example For scenarios where you want persistence across reconnections but automatic cleanup (e.g., Kubernetes pods), you can create durable consumers with `inactive_threshold`: defmodule MyApp.AutoCleanupPullConsumer do use Gnat.Jetstream.PullConsumer def start_link(pod_name) do Jetstream.PullConsumer.start_link(__MODULE__, pod_name) end @impl true def init(pod_name) do consumer = %Gnat.Jetstream.API.Consumer{ stream_name: "ORDERS_STREAM", durable_name: "orders-consumer-#{System.get_env("POD_NAME")}", # Named after pod inactive_threshold: 300_000_000_000, # 5 minutes in nanoseconds filter_subject: "orders.*" } {:ok, nil, connection_name: :gnat, consumer: consumer} end @impl true def handle_message(message, state) do # Process order message with state persistence {:ack, state} end end ## How to supervise A `PullConsumer` is most commonly started under a supervision tree. When we invoke `use Gnat.Jetstream.PullConsumer`, it automatically defines a `child_spec/1` function that allows us to start the pull consumer directly under a supervisor. To start a pull consumer under a supervisor with an initial argument of :example, one may do: children = [ {MyPullConsumer, :example} ] Supervisor.start_link(children, strategy: :one_for_all) While one could also simply pass the `MyPullConsumer` as a child to the supervisor, such as: children = [ MyPullConsumer # Same as {MyPullConsumer, []} ] Supervisor.start_link(children, strategy: :one_for_all) A common approach is to use a keyword list, which allows setting init argument and server options, for example: def start_link(opts) do {initial_state, opts} = Keyword.pop(opts, :initial_state, nil) Gnat.Jetstream.PullConsumer.start_link(__MODULE__, initial_state, opts) end and then you can use `MyPullConsumer`, `{MyPullConsumer, name: :my_consumer}` or even `{MyPullConsumer, initial_state: :example, name: :my_consumer}` as a child specification. `use Gnat.Jetstream.PullConsumer` also accepts a list of options which configures the child specification and therefore how it runs under a supervisor. The generated `child_spec/1` can be customized with the following options: * `:id` - the child specification identifier, defaults to the current module * `:restart` - when the child should be restarted, defaults to `:permanent` * `:shutdown` - how to shut down the child, either immediately or by giving it time to shut down For example: use Gnat.Jetstream.PullConsumer, restart: :transient, shutdown: 10_000 See the "Child specification" section in the `Supervisor` module for more detailed information. The `@doc` annotation immediately preceding `use Jetstream.PullConsumer` will be attached to the generated `child_spec/1` function. ## Name registration A pull consumer is bound to the same name registration rules as GenServers. Read more about it in the `GenServer` documentation. """ @doc """ Invoked when the server is started. `start_link/3` or `start/3` will block until it returns. `init_arg` is the argument term (second argument) passed to `start_link/3`. See `c:Connection.init/1` for more details. """ @callback init(init_arg :: term) :: {:ok, state :: term(), connection_options()} | :ignore | {:stop, reason :: any} @doc """ Invoked to synchronously process a message pulled by the consumer. Depending on the value it returns, the acknowledgement is or is not sent. Only real stream messages reach this callback. JetStream informational status messages (e.g. `100` heartbeat, `404`/`408` pull terminator, `409` leadership change) are intercepted by the consumer and never passed here. See `c:handle_status/2` if you want to observe them. ## ACK actions Possible ACK actions values explained: * `:ack` - acknowledges the message was handled and requests delivery of the next message to the reply subject. * `:nack` - signals that the message will not be processed now and processing can move onto the next message, NAK'd message will be retried. * `:term` - instructs the server to stop redelivery of a message without acknowledging it as successfully processed. * `:noreply` - nothing is sent. You may send later asynchronously an ACK or NACK message using the `Jetstream.ack/1` or `Jetstream.nack/1` and similar functions from `Jetstream` module. ## Example def handle_message(message, state) do IO.inspect(message) {:ack, state} end """ @callback handle_message(message :: Gnat.message(), state :: term()) :: {ack_action, new_state} when ack_action: :ack | :nack | :term | :noreply, new_state: term() @doc """ Invoked after the consumer has been created or verified on the NATS server. This callback is called during connection (and reconnection) after the JetStream consumer has been successfully created or confirmed to exist. It receives the full consumer info map returned by the server, which includes fields like `num_pending` (the number of messages waiting to be delivered). This is useful for detecting the initial state of the consumer. For example, if `num_pending` is `0`, you know there are no existing messages to replay and can mark the consumer as caught up immediately. Returning `{:ok, state}` allows you to update the consumer's state based on the consumer info. This callback is optional. If not implemented, the state is passed through unchanged. ## Example @impl true def handle_connected(consumer_info, state) do if consumer_info.num_pending == 0 do {:ok, mark_as_loaded(state)} else {:ok, state} end end """ @callback handle_connected( consumer_info :: Gnat.Jetstream.API.Consumer.info(), state :: term() ) :: {:ok, new_state :: term()} @doc """ Invoked when the consumer receives an informational JetStream status message instead of a real stream message. JetStream delivers status messages on the same subscription as regular messages — for example a `100` idle heartbeat, a `404`/`408` pull request terminator, or a `409` leadership change. These are not stream records and cannot be acked, so the PullConsumer never forwards them to `c:handle_message/2`. By default they are silently dropped and the consumer continues fetching the next message. Implement this callback if you want to observe them — for example to log a warning on leadership changes, or to track heartbeat arrival. The callback receives the raw `Gnat.message()` (which includes `:status` and optionally `:description`) and the current state. Returning `{:ok, new_state}` updates the state; the consumer then proceeds the same way it would have if the callback had not been defined. This callback is optional. ## Example @impl true def handle_status(%{status: "409", description: description}, state) do Logger.warning("JetStream 409 from consumer: #\{description}") {:ok, state} end def handle_status(_message, state), do: {:ok, state} """ @callback handle_status(message :: Gnat.message(), state :: term()) :: {:ok, new_state :: term()} @optional_callbacks [handle_connected: 2, handle_status: 2] @typedoc """ The pull consumer reference. """ @type consumer :: GenServer.server() @typedoc """ Connection option values used to connect the consumer to NATS server. """ @type connection_option :: {:connection_name, GenServer.server()} | {:stream_name, String.t()} | {:consumer_name, String.t()} | {:consumer, Gnat.Jetstream.API.Consumer.t()} | {:connection_retry_timeout, non_neg_integer()} | {:connection_retries, non_neg_integer()} | {:domain, String.t()} | {:batch_size, pos_integer()} | {:request_expires, non_neg_integer()} @typedoc """ Connection options used to connect the consumer to NATS server. """ @type connection_options :: [connection_option()] defmacro __using__(opts) do quote location: :keep, bind_quoted: [opts: opts] do @behaviour Gnat.Jetstream.PullConsumer unless Module.has_attribute?(__MODULE__, :doc) do @doc """ Returns a specification to start this module under a supervisor. See the "Child specification" section in the `Supervisor` module for more detailed information. """ end @spec child_spec(arg :: GenServer.options()) :: Supervisor.child_spec() def child_spec(arg) do default = %{ id: __MODULE__, start: {__MODULE__, :start_link, [arg]} } Supervisor.child_spec(default, unquote(Macro.escape(opts))) end defoverridable child_spec: 1 end end @doc """ Starts a pull consumer linked to the current process with the given function. This is often used to start the pull consumer as part of a supervision tree. Once the server is started, the `c:init/1` function of the given `module` is called with `init_arg` as its argument to initialize the server. To ensure a synchronized start-up procedure, this function does not return until `c:init/1` has returned. See `GenServer.start_link/3` for more details. """ @spec start_link(module(), init_arg :: term(), options :: GenServer.options()) :: GenServer.on_start() def start_link(module, init_arg, options \\ []) when is_atom(module) and is_list(options) do Connection.start_link( Gnat.Jetstream.PullConsumer.Server, %{module: module, init_arg: init_arg}, options ) end @doc """ Starts a `Jetstream.PullConsumer` process without links (outside of a supervision tree). See `start_link/3` for more information. """ @spec start(module(), init_arg :: term(), options :: GenServer.options()) :: GenServer.on_start() def start(module, init_arg, options \\ []) when is_atom(module) and is_list(options) do Connection.start( Gnat.Jetstream.PullConsumer.Server, %{module: module, init_arg: init_arg}, options ) end @doc """ Closes the pull consumer and stops underlying process. ## Example {:ok, consumer} = PullConsumer.start_link(ExamplePullConsumer, connection_name: :gnat, stream_name: "TEST_STREAM", consumer_name: "TEST_CONSUMER" ) :ok = PullConsumer.close(consumer) """ @spec close(consumer :: consumer()) :: :ok def close(consumer) do Connection.call(consumer, :close) end end ================================================ FILE: lib/gnat/parsec.ex ================================================ defmodule Gnat.Parsec do @moduledoc false defstruct partial: nil import NimbleParsec subject = ascii_string([?!..?~], min: 1) length = integer(min: 1) sid = integer(min: 1) whitespace = ascii_char([32, ?\t]) |> times(min: 1) op_msg = ascii_char([?m, ?M]) |> ascii_char([?s, ?S]) |> ascii_char([?g, ?G]) op_hmsg = ascii_char([?h, ?H]) |> ascii_char([?m, ?M]) |> ascii_char([?s, ?S]) |> ascii_char([?g, ?G]) op_err = ascii_char([?-]) |> ascii_char([?e, ?E]) |> ascii_char([?r, ?R]) |> ascii_char([?r, ?R]) op_info = ascii_char([?i, ?I]) |> ascii_char([?n, ?N]) |> ascii_char([?f, ?F]) |> ascii_char([?o, ?O]) op_ping = ascii_char([?p, ?P]) |> ascii_char([?i, ?I]) |> ascii_char([?n, ?N]) |> ascii_char([?g, ?G]) op_pong = ascii_char([?p, ?P]) |> ascii_char([?o, ?O]) |> ascii_char([?n, ?N]) |> ascii_char([?g, ?G]) op_ok = ascii_char([?+]) |> ascii_char([?o, ?O]) |> ascii_char([?k, ?K]) err = replace(op_err, :err) |> ignore(whitespace) |> ignore(string("'")) |> optional(utf8_string([not: ?'], min: 1)) |> ignore(string("'\r\n")) msg = replace(op_msg, :msg) |> ignore(whitespace) |> concat(subject) |> ignore(whitespace) |> concat(sid) |> ignore(whitespace) |> choice([ subject |> ignore(whitespace) |> concat(length), length ]) |> ignore(string("\r\n")) hmsg = replace(op_hmsg, :hmsg) |> ignore(whitespace) |> concat(subject) |> ignore(whitespace) |> concat(sid) |> ignore(whitespace) |> choice([ subject |> ignore(whitespace) |> concat(length) |> ignore(whitespace) |> concat(length), length |> ignore(whitespace) |> concat(length) ]) |> ignore(string("\r\n")) ok = replace(op_ok |> string("\r\n"), :ok) ping = replace(op_ping |> string("\r\n"), :ping) pong = replace(op_pong |> string("\r\n"), :pong) info = replace(op_info, :info) |> ignore(whitespace) |> utf8_string([not: ?\r], min: 2) |> ignore(string("\r\n")) defparsecp(:command, choice([msg, hmsg, ok, ping, pong, info, err])) def new, do: %__MODULE__{} def parse(%__MODULE__{partial: nil} = state, string) do {partial, commands} = parse_commands(string, []) {%{state | partial: partial}, commands} end def parse(%__MODULE__{partial: partial} = state, string) do {partial, commands} = parse_commands(partial <> string, []) {%{state | partial: partial}, commands} end def parse_commands("", list), do: {nil, Enum.reverse(list)} def parse_commands(str, list) do case parse_command(str) do {:ok, command, rest} -> parse_commands(rest, [command | list]) {:error, partial} -> {partial, Enum.reverse(list)} end end @spec parse_command(binary()) :: {:ok, tuple(), binary()} | {:error, binary()} def parse_command(string) do case command(string) do {:ok, [:msg, subject, sid, length], rest, _, _, _} -> finish_msg(subject, sid, nil, length, rest, string) {:ok, [:msg, subject, sid, reply_to, length], rest, _, _, _} -> finish_msg(subject, sid, reply_to, length, rest, string) {:ok, [:hmsg, subject, sid, header_length, total_length], rest, _, _, _} -> finish_hmsg(subject, sid, nil, header_length, total_length, rest, string) {:ok, [:hmsg, subject, sid, reply_to, header_length, total_length], rest, _, _, _} -> finish_hmsg(subject, sid, reply_to, header_length, total_length, rest, string) {:ok, [atom], rest, _, _, _} -> {:ok, atom, rest} {:ok, [:info, json], rest, _, _, _} -> {:ok, {:info, Jason.decode!(json, keys: :atoms)}, rest} {:ok, [:err, msg], rest, _, _, _} -> {:ok, {:error, msg}, rest} {:error, _, _, _, _, _} -> {:error, string} end end def parse_headers("NATS/1.0" <> rest) do case String.split(rest, "\r\n", parts: 2) do [" " <> status, headers] -> case :cow_http.parse_headers(headers) do {parsed, ""} -> case String.split(status, " ", parts: 2) do [status, description] -> {:ok, status, description, parsed} [status] -> {:ok, status, nil, parsed} end _other -> {:error, "Could not parse headers"} end [_status_line, headers] -> case :cow_http.parse_headers(headers) do {parsed, ""} -> {:ok, nil, nil, parsed} _other -> {:error, "Could not parse headers"} end _other -> {:error, "Could not parse status line"} end end def parse_headers(_other) do {:error, "Could not parse status line prefix"} end def finish_msg(subject, sid, reply_to, length, rest, string) do case rest do <> -> {:ok, {:msg, subject, sid, reply_to, body}, rest} _other -> {:error, string} end end def finish_hmsg(subject, sid, reply_to, header_length, total_length, rest, string) do payload_length = total_length - header_length case rest do <> -> {:ok, status, description, headers} = parse_headers(headers) {:ok, {:hmsg, subject, sid, reply_to, status, description, headers, payload}, rest} _other -> {:error, string} end end end ================================================ FILE: lib/gnat/server.ex ================================================ defmodule Gnat.Server do require Logger @moduledoc """ A behavior for acting as a server for nats messages. You can use this behavior in your own module and then use the `Gnat.ConsumerSupervisor` to listen for and respond to nats messages. ## Example defmodule MyApp.RpcServer do use Gnat.Server def request(%{body: _body}) do {:reply, "hi"} end # defining an error handler is optional, the default one will just call Logger.error for you def error(%{gnat: gnat, reply_to: reply_to}, _error) do Gnat.pub(gnat, reply_to, "Something went wrong and I can't handle your request") end end """ @doc """ Called when a message is received from the broker """ @callback request(message :: Gnat.message()) :: :ok | {:reply, iodata()} | {:error, term()} @doc """ Called when an error occured during the `request/1` If your `request/1` function returned `{:error, term}`, then the `term` you returned will be passed as the second argument. If an exception was raised during your `request/1` function, then the exception will be passed as the second argument. If your `request/1` function returned something other than the supported return types, then its return value will be passed as the second argument. """ @callback error(message :: Gnat.message(), error :: term()) :: :ok | {:reply, iodata()} defmacro __using__(_opts) do quote do @behaviour Gnat.Server def error(_message, error) do require Logger Logger.error( "Gnat.Server encountered an error while handling a request: #{inspect(error)}", type: :gnat_server_error, error: error ) end defoverridable error: 2 end end # The functions below are not documented because they are used internally to run # the callback modules @doc false def execute(module, message) do try do case apply(module, :request, [message]) do :ok -> :done {:reply, data} -> send_reply(message, data) {:error, error} -> execute_error(module, message, error) other -> execute_error(module, message, other) end rescue e -> execute_error(module, message, e) end end @doc false defp execute_error(module, message, error) do try do case apply(module, :error, [message, error]) do :ok -> :done {:reply, data} -> send_reply(message, data) other -> Logger.error( "error handler for #{module} returned something unexpected: #{inspect(other)}", type: :gnat_server_error ) end rescue e -> Logger.error( "error handler for #{module} encountered an error: #{inspect(e)}", type: :gnat_server_error ) end end @doc false def send_reply(%{gnat: gnat, reply_to: return_address}, iodata) when is_binary(return_address) do Gnat.pub(gnat, return_address, iodata) end def send_reply(_other, _iodata) do Logger.error( "Could not send reply because no reply_to was provided with the original message", type: :gnat_server_error ) end end ================================================ FILE: lib/gnat/services/server.ex ================================================ defmodule Gnat.Services.Server do require Logger alias Gnat.Services.{Service, ServiceResponder} @moduledoc """ A behavior for acting as a NATS service Creating a service with this behavior works almost exactly the same as `Gnat.Server`, with the bonus that this service keeps track of requests, errors, processing time, and participates in service discovery and monitoring as defined by the [NATS service protocol](https://github.com/nats-io/nats-architecture-and-design/blob/main/adr/ADR-32.md). ## Example defmodule MyApp.Service do use Gnat.Services.Server # Classic subject matching def request(%{body: _body, topic: "myservice.req"}, _, _) do {:reply, "handled request"} end # Can also match on endpoint or group def request(msg, "add", "calculator") do {:reply, "42"} end # defining an error handler is optional, the default one will just call Logger.error for you def error(%{gnat: gnat, reply_to: reply_to}, _error) do Gnat.pub(gnat, reply_to, "Something went wrong and I can't handle your request") end end """ @doc """ Called when a message is received from the broker. The endpoint on which the message arrived is always supplied. If the endpoint is a member of a group, the group name will also be provided. Automatically increments the request time and processing time stats for this service. """ @callback request(message :: Gnat.message(), endpoint :: String.t(), group :: String.t() | nil) :: :ok | {:reply, iodata()} | {:error, term()} @doc """ Called when an error occured during the `request/1`. Automatically increments the error count and processing time stats for this service. If your `request/1` function returned `{:error, term}`, then the `term` you returned will be passed as the second argument. If an exception was raised during your `request/1` function, then the exception will be passed as the second argument. If your `request/1` function returned something other than the supported return types, then its return value will be passed as the second argument. """ @callback error(message :: Gnat.message(), error :: term()) :: :ok | {:reply, iodata()} defmacro __using__(_opts) do quote do @behaviour Gnat.Services.Server def error(_message, error) do require Logger Logger.error( "Gnat.Server encountered an error while handling a request: #{inspect(error)}", type: :gnat_server_error, error: error ) end defoverridable error: 2 end end @typedoc """ Service configuration is provided as part of the consumer supervisor settings in the `service_definition` field. You can specify _either_ the `subscription_topics` field for a regluar server or the `service_definition` field for a new NATS service. * `name` - The name of the service. Needs to conform to the rules for NATS service names * `version` - A required version number (w/out "v" prefix) conforming to semver rules * `queue_group` - An optional queue group for service subscriptions. If left off, "q" will be used. * `description` - An optional description of the service * `metadata` - An optional string->string map of service metadata * `endpoints` - A required list of service endpoints. All services must have at least one endpoint """ @type service_configuration :: %{ required(:name) => binary(), required(:version) => binary(), required(:endpoints) => [endpoint_configuration()], optional(:description) => binary(), optional(:metadata) => map() } @typedoc """ Each service configuration must contain at least one endpoint. Endpoints can manually specify their subscription subjects or they can be derived from the endpoint name. * `subject` - A specific subject for this endpoint to listen on. If this is not provided, then the endpoint name will be used. * `name` - The required name of the endpoint * `group_name` - An optional group to which this endpoint belongs * `queue_group` - A queue group for this endpoint's subscription. If not supplied, "q" will be used (indicated by protocol spec). * `metadata` - An optional string->string map containing metadata for this endpoint """ @type endpoint_configuration :: %{ required(:name) => binary(), optional(:subject) => binary(), optional(:group_name) => binary(), optional(:queue_group) => binary(), optional(:metadata) => map() } @doc false def execute(_module, %{topic: "$SRV" <> _} = message, service) do ServiceResponder.maybe_respond(message, service) end def execute(module, message, service) do try do endpoint = Map.get(service.subjects, message.topic) %{group_name: group_name, name: endpoint_name} = endpoint telemetry_tags = %{topic: message.topic, endpoint: endpoint_name, group: group_name} case :timer.tc(fn -> apply(module, :request, [message, endpoint_name, group_name]) end) do {_elapsed, :ok} -> :done {elapsed_micros, {:reply, data}} -> send_reply(message, data) :telemetry.execute( [:gnat, :service_request], %{latency: elapsed_micros}, telemetry_tags ) Service.record_request(endpoint, elapsed_micros) {elapsed_micros, {:error, error}} -> execute_error(module, message, error) :telemetry.execute([:gnat, :service_error], %{latency: elapsed_micros}, telemetry_tags) Service.record_error(endpoint, elapsed_micros) other -> execute_error(module, message, other) end rescue e -> execute_error(module, message, e) end end @doc false defp execute_error(module, message, error) do try do case apply(module, :error, [message, error]) do :ok -> :done {:reply, data} -> send_reply(message, data) other -> Logger.error( "error handler for #{module} returned something unexpected: #{inspect(other)}", type: :gnat_server_error ) end rescue e -> Logger.error( "error handler for #{module} encountered an error: #{inspect(e)}", type: :gnat_server_error ) end end @doc false def send_reply(%{gnat: gnat, reply_to: return_address}, iodata) when is_binary(return_address) do Gnat.pub(gnat, return_address, iodata) end def send_reply(_other, _iodata) do Logger.error( "Could not send reply because no reply_to was provided with the original message", type: :gnat_server_error ) end end ================================================ FILE: lib/gnat/services/service.ex ================================================ defmodule Gnat.Services.Service do @moduledoc false @subscription_subject "$SRV.>" # required default, see https://github.com/nats-io/nats-architecture-and-design/blob/main/adr/ADR-32.md#request-handling @default_service_queue_group "q" @idx_requests 1 @idx_errors 2 @idx_processing_time 3 @name_regex ~r/^[a-zA-Z0-9_-]+$/ @version_regex ~r/^(0|[1-9]\d*)\.(0|[1-9]\d*)\.(0|[1-9]\d*)(?:-((?:0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*)(?:\.(?:0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*))*))?(?:\+([0-9a-zA-Z-]+(?:\.[0-9a-zA-Z-]+)*))?$/ alias Gnat.Services.WireProtocol def init(configuration) do with :ok <- validate_configuration(configuration) do service = %{ name: configuration.name, instance_id: :crypto.strong_rand_bytes(12) |> Base.encode64(), description: configuration.description, version: configuration.version, subjects: build_subject_map(configuration.endpoints), started: DateTime.to_iso8601(DateTime.utc_now()), metadata: Map.get(configuration, :metadata, %{}) } {:ok, service} end end def info(service) do endpoint_info = service.subjects |> Map.values() |> Enum.map(&endpoint_info/1) %WireProtocol.InfoResponse{ name: service.name, description: service.description, id: service.instance_id, version: service.version, metadata: service.metadata, endpoints: endpoint_info } end def ping(service) do %WireProtocol.PingResponse{ name: service.name, id: service.instance_id, version: service.version, metadata: service.metadata } end def record_request(%{counters: counters} = _endpoint, elapsed_micros) do :counters.add(counters, @idx_requests, 1) :counters.add(counters, @idx_processing_time, elapsed_micros) end def record_error(%{counters: counters} = _endpoint, elapsed_micros) do :counters.add(counters, @idx_errors, 1) :counters.add(counters, @idx_processing_time, elapsed_micros) end def subscription_topics_with_queue_group(service) do endpoint_subscriptions = service.subjects |> Enum.map(fn {topic, metadata} -> {topic, metadata.queue_group} end) services_subscription = {@subscription_subject, nil} [services_subscription | endpoint_subscriptions] end def stats(service) do endpoint_stats = service.subjects |> Map.values() |> Enum.map(&endpoint_stats/1) %WireProtocol.StatsResponse{ name: service.name, id: service.instance_id, version: service.version, started: service.started, endpoints: endpoint_stats } end defp build_subject_map(endpoints) do Enum.reduce(endpoints, %{}, fn ep, map -> subject = derive_subscription_subject(ep) endpoint = %{ name: ep.name, queue_group: Map.get(ep, :queue_group, @default_service_queue_group), group_name: Map.get(ep, :group_name, nil), metadata: Map.get(ep, :metadata, %{}), subject: subject, counters: :counters.new(3, [:atomics]) } Map.put(map, subject, endpoint) end) end @spec derive_subscription_subject(Gnat.Services.Server.endpoint_configuration()) :: String.t() defp derive_subscription_subject(endpoint) do group_prefix = case Map.get(endpoint, :group_name) do nil -> "" prefix -> "#{prefix}." end subject = case Map.get(endpoint, :subject) do nil -> endpoint.name sub -> sub end "#{group_prefix}#{subject}" end defp endpoint_info(endpoint) do %{ name: endpoint.name, subject: endpoint.subject, metadata: endpoint.metadata, queue_group: endpoint.queue_group } end defp endpoint_stats(%{counters: counters} = endpoint) do micros = :counters.get(counters, @idx_processing_time) nanos = 1000 * micros num_errors = :counters.get(counters, @idx_errors) num_requests = :counters.get(counters, @idx_requests) total_calls = num_errors + num_requests avg = if total_calls > 0 do trunc(ceil(nanos / total_calls)) else 0 end %{ name: endpoint.name, subject: endpoint.subject, num_requests: num_requests, num_errors: num_errors, processing_time: nanos, average_processing_time: avg, queue_group: endpoint.queue_group } end defp validate_configuration(configuration) when is_nil(configuration), do: {:error, ["Service definition cannot be null"]} defp validate_configuration(configuration) when not is_map(configuration), do: {:error, ["Service definition must be a map"]} defp validate_configuration(configuration) do rules = [ {&valid_version?/1, configuration}, {&valid_name?/1, configuration}, {&valid_metadata?/1, Map.get(configuration, :metadata)} ] eprules = configuration.endpoints |> Enum.map(fn ep -> {&valid_endpoint?/1, ep} end) results = (rules ++ eprules) |> Enum.map(fn {pred, input} -> apply(pred, [input]) end) {_good, bad} = Enum.split_with(results, fn e -> e == :ok end) if length(bad) == 0 do :ok else {:error, bad |> Enum.map(fn {:error, m} -> m end) |> Enum.to_list()} end end defp valid_version?(service_definition) do version = Map.get(service_definition, :version) if String.match?(version, @version_regex) do :ok else {:error, "Version '#{version}' does not conform to semver specification"} end end defp valid_name?(service_definition) do name = Map.get(service_definition, :name) if String.match?(name, @name_regex) do :ok else {:error, "Service name '#{name}' is invalid. Check for illegal characters"} end end defp valid_metadata?(nil), do: :ok defp valid_metadata?(md) do bads = Enum.filter(md, fn {k, v} -> !is_binary(k) or !is_binary(v) end) |> length() if bads == 0 do :ok else {:error, "At least one key or value found in metadata that was not a string"} end end defp valid_endpoint?(endpoint_definition) do name = Map.get(endpoint_definition, :name) with true <- String.match?(name, @name_regex), :ok <- valid_metadata?(Map.get(endpoint_definition, :metadata)) do :ok else false -> {:error, "Endpoint name '#{name}' is not valid"} e -> e end end end ================================================ FILE: lib/gnat/services/service_responder.ex ================================================ defmodule Gnat.Services.ServiceResponder do @moduledoc false require Logger alias Gnat.Services.Service @op_ping "PING" @op_stats "STATS" @op_info "INFO" def maybe_respond(%{topic: topic} = message, service) do case String.split(topic, ".") do ["$SRV", @op_ping | rest] -> handle_ping(rest, service, message) ["$SRV", @op_info | rest] -> handle_info(rest, service, message) ["$SRV", @op_stats | rest] -> handle_stats(rest, service, message) _other -> Logger.error("ServiceResponder received unexpected message #{topic}") end end defp handle_ping(tail, service, %{reply_to: rt, gnat: gnat}) do if should_respond?(tail, service.name, service.instance_id) do body = Service.ping(service) |> Jason.encode!() Gnat.pub(gnat, rt, body) end end defp handle_info(tail, service, %{reply_to: rt, gnat: gnat}) do if should_respond?(tail, service.name, service.instance_id) do body = Service.info(service) |> Jason.encode!() Gnat.pub(gnat, rt, body) end end defp handle_stats(tail, service, %{reply_to: rt, gnat: gnat}) do if should_respond?(tail, service.name, service.instance_id) do body = Service.stats(service) |> Jason.encode!() Gnat.pub(gnat, rt, body) end end @spec should_respond?(list, String.t(), String.t()) :: boolean() defp should_respond?(tail, service_name, instance_id) do case tail do [] -> true [^service_name] -> true [^service_name, ^instance_id] -> true _ -> false end end end ================================================ FILE: lib/gnat/services/wire_protocol.ex ================================================ defmodule Gnat.Services.WireProtocol do @moduledoc false defmodule InfoResponse do @moduledoc false @info_response_type "io.nats.micro.v1.info_response" @type endpoint :: %{ name: String.t(), subject: String.t(), metadata: map(), queue_group: String.t() } @type t :: %__MODULE__{ name: String.t(), id: String.t(), version: String.t(), description: String.t(), metadata: map(), endpoints: [endpoint()], type: String.t() } @derive Jason.Encoder @enforce_keys [:name, :id, :version] defstruct [ :name, :id, :version, :metadata, :description, endpoints: [], type: @info_response_type ] end defmodule PingResponse do @moduledoc false @ping_response_type "io.nats.micro.v1.ping_response" @type t :: %__MODULE__{ name: String.t(), id: String.t(), version: String.t(), metadata: map() } @derive Jason.Encoder @enforce_keys [:name, :id, :version] defstruct [ :name, :id, :version, :metadata, type: @ping_response_type ] end defmodule StatsResponse do @moduledoc false @stats_response_type "io.nats.micro.v1.stats_response" @type endpoint :: %{ name: String.t(), subject: String.t(), num_requests: integer, num_errors: integer, last_error: String.t(), processing_time: integer, average_processing_time: integer, queue_group: String.t(), data: map() } @type t :: %__MODULE__{ name: String.t(), id: String.t(), version: String.t(), metadata: map(), started: String.t(), endpoints: [endpoint()], type: String.t() } @derive Jason.Encoder @enforce_keys [:name, :id] defstruct [ :name, :id, :version, :metadata, :started, :endpoints, type: @stats_response_type ] end end ================================================ FILE: lib/gnat.ex ================================================ # State transitions: # :waiting_for_message => receive PING, send PONG => :waiting_for_message # :waiting_for_message => receive MSG... -> :waiting_for_message defmodule Gnat do @moduledoc """ The primary interface for interacting with NATS """ use GenServer require Logger alias Gnat.{Command, Parsec} @type t :: GenServer.server() @type headers :: [{binary(), iodata()}] @typedoc """ A message received from NATS will be delivered to your process in this form. Please note that the `:reply_to` and `:headers` keys are optional. They will only be present if the message was received from the NATS server with headers or a `reply_to` topic * `gnat` - The Gnat connection * `topic` - The topic on which the message arrived * `body` - The raw payload of the message * `sid` - The subscription ID corresponding to this message. You generally won't need to use this value directly. * `reply_to` - A topic supplied for expected replies * `headers` - A set of NATS message headers on the message * `status` - Similar to an HTTP status, this is present for messages with headers and can indicate the specific purpose of a message. Example `status: "408"` * `description` - A string description of the `status` """ @type message :: %{ required(:gnat) => t(), required(:topic) => binary(), required(:body) => iodata(), required(:sid) => non_neg_integer(), optional(:reply_to) => binary(), optional(:headers) => headers(), optional(:status) => String.t(), optional(:description) => String.t() } @type sent_message :: {:msg, message()} @typedoc """ * `connection_timeout` - limits how long it can take to establish a connection to a server * `host` - The location of the NATS server * `ping_interval` - The number of milliseconds between sending PING messages to the server to check the health of our connection * `port` - The port the NATS server is listening on * `ssl_opts` - Options for connecting over SSL * `tcp_opts` - Options for connecting over TCP * `tls` - If the server should use a TLS connection * `inbox_prefix` - Prefix to use for the message inbox of this connection * `no_responders` - Enable the no responders behavior (see `Gnat.request/4`) """ @type connection_settings :: %{ optional(:connection_timeout) => non_neg_integer(), optional(:host) => binary(), optional(:inbox_prefix) => binary(), optional(:ping_interval) => non_neg_integer(), optional(:port) => non_neg_integer(), optional(:ssl_opts) => list(), optional(:tcp_opts) => list(), optional(:tls) => boolean(), optional(:no_responders) => boolean() } @typedoc """ [Info Protocol](https://docs.nats.io/reference/reference-protocols/nats-protocol#info) * `client_id` - An optional unsigned integer (64 bits) representing the internal client identifier in the server. This can be used to filter client connections in monitoring, correlate with error logs, etc... * `client_ip` - The IP address the client is connecting from * `cluster` - The name of the cluster if any * `cluster_dynamic` - If the cluster is dynamic * `connect_urls` - An optional list of server urls that a client can connect to. * `ws_connect_urls` - An optional list of server urls that a websocket client can connect to. * `git_commit` - The git commit associated with this NATS version * `go` - The version of golang the NATS server was built with * `headers` - If messages can have headers in them * `host` - The IP address used to start the NATS server, by default this will be 0.0.0.0 and can be configured with -client_advertise host:port * `jetstream` - If the server is using JetStream features * `max_payload` - Maximum payload size, in bytes, that the server will accept from the client * `port` - The port number the NATS server is configured to listen on * `proto` - An integer indicating the protocol version of the server. The server version 1.2.0 sets this to 1 to indicate that it supports the "Echo" feature. * `server_id` - The unique identifier of the NATS server * `server_name` - A name for the server * `version` - The version of the NATS server * `ldm` - If the server supports Lame Duck Mode notifications, and the current server has transitioned to lame duck, ldm will be set to true. * `auth_required` - If this is set, then the client should try to authenticate upon connect. * `tls_required` - If this is set, then the client must perform the TLS/1.2 handshake. Note, this used to be ssl_required and has been updated along with the protocol from SSL to TLS. * `tls_verify` - If this is set, the client must provide a valid certificate during the TLS handshake. * `tls_available` - If the server can use TLS """ @type server_info :: %{ :client_id => non_neg_integer(), :client_ip => binary(), optional(:ip) => binary(), optional(:cluster) => binary(), optional(:cluster_dynamic) => boolean(), optional(:connect_urls) => list(binary()), optional(:ws_connect_urls) => list(binary()), optional(:git_commit) => binary(), :go => binary(), :headers => boolean(), :host => binary(), optional(:jetstream) => binary(), :max_payload => integer(), :port => non_neg_integer(), :proto => integer(), :server_id => binary(), :server_name => binary(), :version => binary(), optional(:ldm) => boolean(), optional(:tls_verify) => boolean(), optional(:tls_available) => boolean(), optional(:tls_required) => boolean(), optional(:auth_required) => boolean() } @default_connection_settings %{ host: ~c"localhost", ping_interval: 10_000, port: 4222, tcp_opts: [:binary], connection_timeout: 3_000, ssl_opts: [], tls: false, inbox_prefix: "_INBOX.", no_responders: false } @request_sid 0 @doc """ Starts a connection to a nats broker ``` {:ok, gnat} = Gnat.start_link(%{host: "127.0.0.1", port: 4222}) # if the server requires TLS you can start a connection with: {:ok, gnat} = Gnat.start_link(%{host: "127.0.0.1", port: 4222, tls: true}) # if the server requires TLS and a client certificate you can start a connection with: {:ok, gnat} = Gnat.start_link(%{tls: true, ssl_opts: [certfile: "client-cert.pem", keyfile: "client-key.pem"]}) # you can customize default "_INBOX." inbox prefix with: {:ok, gnat} = Gnat.start_link(%{host: "127.0.0.1", port: 4222, inbox_prefix: "my_prefix._INBOX."}) # you can use IPv6 addresses too {:ok, gnat} = Gnat.start_link(%{host: "::1", port: 4222, tcp_opts: [:inet6, :binary]}) ``` You can also pass arbitrary SSL or TCP options in the `tcp_opts` and `ssl_opts` keys. If you pass custom TCP options please include `:binary`. Gnat uses binary matching to parse messages. The final `opts` argument will be passed to the `GenServer.start_link` call so you can pass things like `[name: :gnat_connection]`. """ @spec start_link(connection_settings(), keyword()) :: GenServer.on_start() def start_link(connection_settings \\ %{}, opts \\ []) do GenServer.start_link(__MODULE__, connection_settings, opts) end @doc """ Gracefully shuts down a connection ``` {:ok, gnat} = Gnat.start_link() :ok = Gnat.stop(gnat) ``` """ @spec stop(t()) :: :ok def stop(pid), do: GenServer.call(pid, :stop) @doc """ Subscribe to a topic Supported options: * queue_group: a string that identifies which queue group you want to join By default each subscriber will receive a copy of every message on the topic. When a queue_group is supplied messages will be spread among the subscribers in the same group. (see [nats queueing](https://nats.io/documentation/concepts/nats-queueing/)) The subscribed process will begin receiving messages with a structure of `t:sent_message/0` ``` {:ok, gnat} = Gnat.start_link() {:ok, subscription} = Gnat.sub(gnat, self(), "topic") receive do {:msg, %{topic: "topic", body: body}} -> IO.puts "Received: \#\{body\}" end ``` """ @spec sub(t(), pid(), String.t(), keyword()) :: {:ok, non_neg_integer()} | {:ok, String.t()} | {:error, String.t()} def sub(pid, subscriber, topic, opts \\ []) do start = :erlang.monotonic_time() result = GenServer.call(pid, {:sub, subscriber, topic, opts}) latency = :erlang.monotonic_time() - start :telemetry.execute([:gnat, :sub], %{latency: latency}, %{topic: topic}) result end @doc """ Publish a message ``` {:ok, gnat} = Gnat.start_link() :ok = Gnat.pub(gnat, "characters", "Ron Swanson") ``` If you want to provide a reply address to receive a response you can pass it as an option. [See request-reply pattern](https://docs.nats.io/nats-concepts/core-nats/reqreply). ``` {:ok, gnat} = Gnat.start_link() :ok = Gnat.pub(gnat, "characters", "Star Lord", reply_to: "me") ``` If you want to publish a message with headers you can pass the `:headers` key in the `opts` like this. ``` {:ok, gnat} = Gnat.start_link() :ok = Gnat.pub(gnat, "listen", "Yo", headers: [{"foo", "bar"}]) ``` Headers must be passed as a `t:headers()` value (a list of tuples). Sending and parsing headers has more overhead than typical nats messages (see [the Nats 2.2 release notes for details](https://docs.nats.io/whats_new_22#message-headers)), so only use them when they are really valuable. """ @spec pub(t(), String.t(), binary(), keyword()) :: :ok def pub(pid, topic, message, opts \\ []) do start = :erlang.monotonic_time() opts = prepare_headers(opts) result = GenServer.call(pid, {:pub, topic, message, opts}) latency = :erlang.monotonic_time() - start :telemetry.execute([:gnat, :pub], %{latency: latency}, %{topic: topic}) result end @doc """ Send a request and listen for a response synchronously Following the nats [request-reply pattern](https://docs.nats.io/nats-concepts/core-nats/reqreply) this function generates a one-time topic to receive replies and then sends a message to the provided topic. Supported options: * `receive_timeout` - An integer number of milliseconds to wait for a response. Defaults to 60_000 * `headers` - A set of headers you want to send with the request (see `Gnat.pub/4`) ``` {:ok, gnat} = Gnat.start_link() case Gnat.request(gnat, "i_can_haz_cheezburger", "plZZZZ?!?!?") do {:ok, %{body: delicious_cheezburger}} -> :yum {:error, :timeout} -> :sad_cat end ``` ## No Responders If you send a request to a topic that has no registered listeners, it is sometimes convenient to find out right away, rather than waiting for a timeout to occur. In order to support this use-case, you can start your Gnat connection with the `no_responders: true` option and this function will return very quickly with an `{:error, :no_responders}` value. This behavior also works with `request_multi/4` """ @spec request(t(), String.t(), binary(), keyword()) :: {:ok, message} | {:error, :timeout} | {:error, :no_responders} def request(pid, topic, body, opts \\ []) do start = :erlang.monotonic_time() receive_timeout = Keyword.get(opts, :receive_timeout, 60_000) req = %{recipient: self(), body: body, topic: topic} opts = prepare_headers(opts) req = case Keyword.get(opts, :headers) do nil -> req headers -> Map.put(req, :headers, headers) end {:ok, subscription} = GenServer.call(pid, {:request, req}) response = receive_request_response(subscription, receive_timeout) :ok = unsub(pid, subscription) latency = :erlang.monotonic_time() - start :telemetry.execute([:gnat, :request], %{latency: latency}, %{topic: topic}) response end @doc """ Send a request and listen for multiple responses synchronously This function makes it easy to do a scatter-gather operation where you wait for a limited time and optionally a maximum number of replies. Supported options: * `receive_timeout` - An integer number of milliseconds to wait for responses. Defaults to 60_000 * `max_messages` - An integer number of messages to listen for. Defaults to -1 meaning unlimited * `headers` - A set of headers you want to send with the request (see `Gnat.pub/4`) ``` {:ok, gnat} = Gnat.start_link() {:ok, responses} = Gnat.request_multi(gnat, "i_can_haz_fries", "plZZZZZ!?!?", max_messages: 5) Enum.count(responses) #=> 5 ``` """ @spec request_multi(t(), String.t(), binary(), keyword()) :: {:ok, list(message())} | {:error, :no_responders} def request_multi(pid, topic, body, opts \\ []) do start = :erlang.monotonic_time() receive_timeout_ms = Keyword.get(opts, :receive_timeout, 60_000) expiration = System.monotonic_time(:millisecond) + receive_timeout_ms max_messages = Keyword.get(opts, :max_messages, -1) req = %{recipient: self(), body: body, topic: topic} opts = prepare_headers(opts) req = case Keyword.get(opts, :headers) do nil -> req headers -> Map.put(req, :headers, headers) end {:ok, subscription} = GenServer.call(pid, {:request, req}) result = case receive_multi_request_responses(subscription, expiration, max_messages) do {:error, :no_responders} -> {:error, :no_responders} responses when is_list(responses) -> {:ok, responses} end :ok = unsub(pid, subscription) latency = :erlang.monotonic_time() - start :telemetry.execute([:gnat, :request_multi], %{latency: latency}, %{topic: topic}) result end @doc """ Unsubscribe from a topic Supported options: * `max_messages` - Number of messages to be received before automatically unsubscribed This correlates to the [UNSUB](https://docs.nats.io/reference/reference-protocols/nats-protocol#unsub) command in the nats protocol. By default the unsubscribe is affected immediately, but an optional `max_messages` value can be provided which will allow `max_messages` to be received before affecting the unsubscribe. This is especially useful for request/reply patterns. ``` {:ok, gnat} = Gnat.start_link() {:ok, subscription} = Gnat.sub(gnat, self(), "my_inbox") :ok = Gnat.unsub(gnat, subscription) # OR :ok = Gnat.unsub(gnat, subscription, max_messages: 2) ``` """ @spec unsub(t(), non_neg_integer() | String.t(), keyword()) :: :ok def unsub(pid, sid, opts \\ []) do start = :erlang.monotonic_time() result = GenServer.call(pid, {:unsub, sid, opts}) :telemetry.execute([:gnat, :unsub], %{latency: :erlang.monotonic_time() - start}) result end @doc """ Kept just for backward compatibility for now """ @deprecated "Pinging is handled internally by the connection, this function is now a no-op until we remove the function in the 2.x series" def ping(_pid) do :ok end @doc "Get the number of active subscriptions" @spec active_subscriptions(t()) :: {:ok, non_neg_integer()} def active_subscriptions(pid) do GenServer.call(pid, :active_subscriptions) end @doc """ Get information about the NATS server the connection is for """ @spec server_info(t()) :: server_info() def server_info(name) do GenServer.call(name, :server_info) end @impl GenServer def init(connection_settings) do connection_settings = Map.merge(@default_connection_settings, connection_settings) case Gnat.Handshake.connect(connection_settings) do {:ok, socket, server_info} -> schedule_ping_check(connection_settings) parser = Parsec.new() request_inbox_prefix = Map.fetch!(connection_settings, :inbox_prefix) <> "#{nuid()}." state = %{ socket: socket, connection_settings: connection_settings, server_info: server_info, next_sid: 1, receivers: %{}, parser: parser, request_receivers: %{}, request_inbox_prefix: request_inbox_prefix, waiting_on_pong: false } state = create_request_subscription(state) {:ok, state} {:error, reason} -> {:stop, reason} end end @impl GenServer def handle_info(:ping_check, %{waiting_on_pong: true} = state) do error_message = "Closing connection because we did not receive a PONG back within #{state.connection_settings.ping_interval}ms" Logger.error(error_message) {:stop, error_message} end def handle_info(:ping_check, %{waiting_on_pong: false} = state) do :ok = socket_write(state, "PING\r\n") schedule_ping_check(state.connection_settings) {:noreply, %{state | waiting_on_pong: true}} end def handle_info({:tcp, socket, data}, %{socket: socket} = state) do data_packets = receive_additional_tcp_data(socket, [data], 10) new_state = Enum.reduce(data_packets, state, fn data, %{parser: parser} = state -> {new_parser, messages} = Parsec.parse(parser, data) new_state = %{state | parser: new_parser} Enum.reduce(messages, new_state, &process_message/2) end) {:noreply, new_state} end def handle_info({:ssl, socket, data}, state) do handle_info({:tcp, socket, data}, state) end def handle_info({:tcp_closed, _}, state) do {:stop, "connection closed", state} end def handle_info({:ssl_closed, _}, state) do {:stop, "connection closed", state} end def handle_info({:tcp_error, _, reason}, state) do {:stop, "tcp transport error #{inspect(reason)}", state} end def handle_info({:DOWN, _ref, :process, pid, _reason}, state) do {sid, _receiver} = Enum.find(state.receivers, fn {_sid, receiver} -> receiver.recipient == pid end) state = unsub_sid(sid, [], state) {:noreply, state} end def handle_info(other, state) do Logger.error("#{__MODULE__} received unexpected message: #{inspect(other)}") {:noreply, state} end @impl GenServer def handle_call(:stop, _from, state) do socket_close(state) {:stop, :normal, :ok, state} end def handle_call({:sub, receiver, topic, opts}, _from, %{next_sid: sid} = state) do sub = Command.build(:sub, topic, sid, opts) :ok = socket_write(state, sub) ref = Process.monitor(receiver) next_state = add_subscription_to_state(state, sid, receiver, ref) |> Map.put(:next_sid, sid + 1) {:reply, {:ok, sid}, next_state} end def handle_call({:pub, topic, message, opts}, from, state) do commands = [Command.build(:pub, topic, message, opts)] froms = [from] {commands, froms} = receive_additional_pubs(commands, froms, 10) :ok = socket_write(state, commands) Enum.each(froms, fn from -> GenServer.reply(from, :ok) end) {:noreply, state} end def handle_call({:request, request}, _from, state) do inbox = make_new_inbox(state) new_state = %{ state | request_receivers: Map.put(state.request_receivers, inbox, request.recipient) } pub = case request do %{headers: headers} -> Command.build(:pub, request.topic, request.body, headers: headers, reply_to: inbox) _ -> Command.build(:pub, request.topic, request.body, reply_to: inbox) end :ok = socket_write(new_state, [pub]) {:reply, {:ok, inbox}, new_state} end # When the SID is a string, it's a topic, which is used as a key in the request receiver map. def handle_call({:unsub, topic, _opts}, _from, state) when is_binary(topic) do if Map.has_key?(state.request_receivers, topic) do request_receivers = Map.delete(state.request_receivers, topic) new_state = %{state | request_receivers: request_receivers} {:reply, :ok, new_state} else {:reply, :ok, state} end end def handle_call({:unsub, sid, opts}, _from, state) do state = unsub_sid(sid, opts, state) {:reply, :ok, state} end def handle_call({:ping, pinger}, _from, state) do :ok = socket_write(state, "PING\r\n") {:reply, :ok, Map.put(state, :pinger, pinger)} end def handle_call(:active_subscriptions, _from, state) do active_subscriptions = Enum.count(state.receivers) {:reply, {:ok, active_subscriptions}, state} end def handle_call(:server_info, _from, state) do {:reply, state.server_info, state} end defp unsub_sid(sid, opts, state) do case Map.get(state.receivers, sid) do nil -> state %{monitor_ref: ref} -> command = Command.build(:unsub, sid, opts) :ok = socket_write(state, command) Process.demonitor(ref) state = cleanup_subscription_from_state(state, sid, opts) state end end defp create_request_subscription(%{request_inbox_prefix: request_inbox_prefix} = state) do # Example: "_INBOX.Jhf7AcTGP3x4dAV9.*" wildcard_inbox_topic = request_inbox_prefix <> "*" sub = Command.build(:sub, wildcard_inbox_topic, @request_sid, []) :ok = socket_write(state, [sub]) add_subscription_to_state(state, @request_sid, self(), nil) end defp make_new_inbox(%{request_inbox_prefix: prefix}), do: prefix <> nuid() defp nuid(), do: :crypto.strong_rand_bytes(12) |> Base.encode64() defp prepare_headers(opts) do if Keyword.has_key?(opts, :headers) do headers = :cow_http.headers(Keyword.get(opts, :headers)) Keyword.put(opts, :headers, headers) else opts end end defp socket_close(%{socket: socket, connection_settings: %{tls: true}}), do: :ssl.close(socket) defp socket_close(%{socket: socket}), do: :gen_tcp.close(socket) defp socket_write(%{socket: socket, connection_settings: %{tls: true}}, iodata) do :ssl.send(socket, iodata) end defp socket_write(%{socket: socket}, iodata), do: :gen_tcp.send(socket, iodata) defp add_subscription_to_state(%{receivers: receivers} = state, sid, pid, ref) do receivers = Map.put(receivers, sid, %{recipient: pid, monitor_ref: ref, unsub_after: :infinity}) %{state | receivers: receivers} end defp cleanup_subscription_from_state(%{receivers: receivers} = state, sid, []) do receivers = Map.delete(receivers, sid) %{state | receivers: receivers} end defp cleanup_subscription_from_state(%{receivers: receivers} = state, sid, max_messages: n) do receivers = put_in(receivers, [sid, :unsub_after], n) %{state | receivers: receivers} end defp process_message({:info, server_info}, state) do %{state | server_info: server_info} end defp process_message({:msg, topic, @request_sid, reply_to, body}, state) do if Map.has_key?(state.request_receivers, topic) do send( state.request_receivers[topic], {:msg, %{topic: topic, body: body, reply_to: reply_to, gnat: self()}} ) state else Logger.error("#{__MODULE__} got a response for a request, but that is no longer registered") state end end defp process_message({:msg, topic, sid, reply_to, body}, state) do unless is_nil(state.receivers[sid]) do :telemetry.execute([:gnat, :message_received], %{count: 1}, %{topic: topic}) send( state.receivers[sid].recipient, {:msg, %{topic: topic, body: body, reply_to: reply_to, sid: sid, gnat: self()}} ) update_subscriptions_after_delivering_message(state, sid) else Logger.error("#{__MODULE__} got message for sid #{sid}, but that is no longer registered") state end end defp process_message( {:hmsg, topic, @request_sid, reply_to, status, description, headers, body}, state ) do if Map.has_key?(state.request_receivers, topic) do map = %{ topic: topic, body: body, reply_to: reply_to, gnat: self(), headers: headers, status: status, description: description } send(state.request_receivers[topic], {:msg, map}) state else Logger.error("#{__MODULE__} got a response for a request, but that is no longer registered") state end end defp process_message({:hmsg, topic, sid, reply_to, status, description, headers, body}, state) do unless is_nil(state.receivers[sid]) do :telemetry.execute([:gnat, :message_received], %{count: 1}, %{topic: topic}) map = %{ topic: topic, body: body, reply_to: reply_to, sid: sid, gnat: self(), headers: headers, status: status, description: description } send(state.receivers[sid].recipient, {:msg, map}) update_subscriptions_after_delivering_message(state, sid) else Logger.error("#{__MODULE__} got message for sid #{sid}, but that is no longer registered") state end end defp process_message(:ping, state) do socket_write(state, "PONG\r\n") state end defp process_message(:pong, state) do %{state | waiting_on_pong: false} end defp process_message({:error, message}, state) do :error_logger.error_report( type: :gnat_error_from_broker, message: message ) state end defp receive_additional_pubs(commands, froms, 0), do: {commands, froms} defp receive_additional_pubs(commands, froms, how_many_more) do receive do {:"$gen_call", from, {:pub, topic, message, opts}} -> commands = [Command.build(:pub, topic, message, opts) | commands] froms = [from | froms] receive_additional_pubs(commands, froms, how_many_more - 1) after 0 -> {commands, froms} end end defp receive_additional_tcp_data(_socket, packets, 0), do: Enum.reverse(packets) defp receive_additional_tcp_data(socket, packets, n) do receive do {:tcp, ^socket, data} -> receive_additional_tcp_data(socket, [data | packets], n - 1) after 0 -> Enum.reverse(packets) end end defp update_subscriptions_after_delivering_message(%{receivers: receivers} = state, sid) do receivers = case get_in(receivers, [sid, :unsub_after]) do :infinity -> receivers 1 -> Map.delete(receivers, sid) n -> put_in(receivers, [sid, :unsub_after], n - 1) end %{state | receivers: receivers} end defp receive_multi_request_responses(_sub, _exp, 0), do: [] defp receive_multi_request_responses(subscription, expiration, max_messages) do timeout = expiration - :erlang.monotonic_time(:millisecond) cond do timeout < 1 -> [] true -> case receive_request_response(subscription, timeout) do {:error, :no_responders} -> {:error, :no_responders} {:error, :timeout} -> [] {:ok, msg} -> [msg | receive_multi_request_responses(subscription, expiration, max_messages - 1)] end end end defp receive_request_response(subscription, timeout) do receive do {:msg, %{topic: ^subscription, status: "503"}} -> {:error, :no_responders} {:msg, %{topic: ^subscription} = msg} -> {:ok, msg} after timeout -> {:error, :timeout} end end defp schedule_ping_check(connection_settings) do Process.send_after(self(), :ping_check, connection_settings.ping_interval) end end ================================================ FILE: mix.exs ================================================ defmodule Gnat.Mixfile do use Mix.Project @source_url "https://github.com/nats-io/nats.ex" @version "1.14.0" def project do [ app: :gnat, version: @version, elixir: "~> 1.14", elixirc_paths: elixirc_paths(Mix.env()), build_embedded: Mix.env() == :prod, start_permanent: Mix.env() == :prod, package: package(), propcheck: [counter_examples: "test/counter_examples"], dialyzer: [ ignore_warnings: ".dialyzer_ignore.exs", plt_file: {:no_warn, "priv/plts/project.plt"}, plt_core_path: "priv/plts/core.plt" ], deps: deps(), docs: docs() ] end def application do [extra_applications: [:logger, :ssl]] end defp deps do [ {:benchee, "~> 1.0", only: :dev}, {:cowlib, "~> 2.0"}, {:dialyxir, "~> 1.0", only: [:dev, :test], runtime: false}, {:ex_doc, "~> 0.36", only: :dev}, {:jason, "~> 1.1"}, {:connection, "~> 1.1"}, {:nimble_parsec, "~> 0.5 or ~> 1.0"}, {:nkeys, "~> 0.2"}, {:propcheck, "~> 1.0", only: :test}, {:telemetry, "~> 0.4 or ~> 1.0"} ] end defp docs do [ main: "readme", logo: "nats-icon-color.svg", source_ref: "v#{@version}", source_url: @source_url, extras: [ "README.md", "docs/js/introduction/overview.md", "docs/js/introduction/getting_started.md", "docs/js/guides/managing.md", "docs/js/guides/push_based_consumer.md", "docs/js/guides/broadway.md", "CHANGELOG.md" ], groups_for_extras: [ "JetStream Introduction": Path.wildcard("docs/js/introduction/*.md"), "JetStream Guides": Path.wildcard("docs/js/guides/*.md") ] ] end defp elixirc_paths(:test), do: ["lib", "test/support"] defp elixirc_paths(_), do: ["lib"] defp package do [ description: "A nats client in pure elixir. Resilience, Performance, Ease-of-Use.", licenses: ["MIT"], links: %{ "Changelog" => "#{@source_url}/blob/master/CHANGELOG.md", "Github" => @source_url }, maintainers: [ "Jon Carstens", "Devin Christensen", "Dave Hackett", "Steve Newell", "Michael Ries", "Garrett Thornburg", "Masahiro Tokioka", "Kevin Hoffman" ], exclude_patterns: ["priv/plts"] ] end end ================================================ FILE: scripts/cluster/cluster.sh ================================================ #!/usr/bin/env bash # Minimal control script for a 3-node local nats cluster used for # manually exercising PullConsumer failover behavior. # # Usage: # scripts/cluster/cluster.sh start # start all 3 # scripts/cluster/cluster.sh start n1 # start just n1 # scripts/cluster/cluster.sh stop # stop all 3 (SIGTERM) # scripts/cluster/cluster.sh stop n2 # stop just n2 # scripts/cluster/cluster.sh kill n2 # SIGKILL n2 (hard fail) # scripts/cluster/cluster.sh status # show what's running # scripts/cluster/cluster.sh clean # stop all + wipe data dirs set -euo pipefail SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" REPO_ROOT="$(cd "$SCRIPT_DIR/../.." && pwd)" PID_DIR="$SCRIPT_DIR/pids" LOG_DIR="$SCRIPT_DIR/logs" DATA_DIR="$SCRIPT_DIR/data" mkdir -p "$PID_DIR" "$LOG_DIR" "$DATA_DIR" nodes=(n1 n2 n3) port_for() { case "$1" in n1) echo 4223 ;; n2) echo 4224 ;; n3) echo 4225 ;; *) echo "unknown node: $1" >&2; return 1 ;; esac } is_running() { local node=$1 local pidfile="$PID_DIR/$node.pid" [[ -f "$pidfile" ]] || return 1 local pid pid=$(cat "$pidfile") kill -0 "$pid" 2>/dev/null } start_node() { local node=$1 if is_running "$node"; then echo "$node already running (pid $(cat "$PID_DIR/$node.pid"))" return 0 fi local conf="$SCRIPT_DIR/$node.conf" local log="$LOG_DIR/$node.log" local pidfile="$PID_DIR/$node.pid" # Run from repo root so the relative store_dir in the conf resolves. ( cd "$REPO_ROOT" nohup nats-server -c "$conf" >"$log" 2>&1 & echo $! > "$pidfile" ) sleep 0.3 if is_running "$node"; then echo "$node started (pid $(cat "$pidfile"), port $(port_for "$node"), log $log)" else echo "$node failed to start — check $log" >&2 return 1 fi } stop_node() { local node=$1 local signal=${2:-TERM} if ! is_running "$node"; then echo "$node not running" rm -f "$PID_DIR/$node.pid" return 0 fi local pid pid=$(cat "$PID_DIR/$node.pid") kill "-$signal" "$pid" echo "$node sent SIG$signal (pid $pid)" rm -f "$PID_DIR/$node.pid" } status() { for node in "${nodes[@]}"; do if is_running "$node"; then echo "$node RUNNING pid=$(cat "$PID_DIR/$node.pid") port=$(port_for "$node")" else echo "$node stopped port=$(port_for "$node")" fi done } cmd=${1:-} shift || true targets=() if (( $# > 0 )); then targets=("$@") else targets=("${nodes[@]}") fi case "$cmd" in start) for n in "${targets[@]}"; do start_node "$n"; done ;; stop) for n in "${targets[@]}"; do stop_node "$n" TERM; done ;; kill) for n in "${targets[@]}"; do stop_node "$n" KILL; done ;; status) status ;; clean) for n in "${nodes[@]}"; do stop_node "$n" KILL || true; done rm -rf "$DATA_DIR" "$LOG_DIR" "$PID_DIR" echo "cleaned data, logs, pids" ;; *) echo "usage: $0 {start|stop|kill|status|clean} [n1|n2|n3]..." >&2 exit 1 ;; esac ================================================ FILE: scripts/cluster/driver.exs ================================================ # Manual failover driver for the 3-node cluster under scripts/cluster/. # # Usage (from repo root, after `scripts/cluster/cluster.sh start`): # # mix run --no-halt scripts/cluster/driver.exs # # The script: # * connects to 2 of the 3 cluster nodes via Gnat.ConnectionSupervisor # * creates (if needed) a 3-replica stream "FAILOVER_STREAM" and a durable # consumer "FAILOVER_CONSUMER" # * starts a PullConsumer that logs every delivery # * publishes a numbered message every second # # Watch the logs, then in another terminal run: # scripts/cluster/cluster.sh kill n1 # scripts/cluster/cluster.sh start n1 # scripts/cluster/cluster.sh kill n2 # …and verify the consumer keeps receiving messages. require Logger Logger.configure(level: :info) alias Gnat.Jetstream.API.{Consumer, Stream} stream_name = "FAILOVER_STREAM" consumer_name = "FAILOVER_CONSUMER" subject = "failover.test" # Deliberately point at only 2 of the 3 nodes so we can prove the connection # can still reach the cluster even when one of the listed nodes is down. connection_settings = [ %{host: ~c"127.0.0.1", port: 4223}, %{host: ~c"127.0.0.1", port: 4224} ] {:ok, _sup} = Gnat.ConnectionSupervisor.start_link( %{ name: :gnat, backoff_period: 1_000, connection_settings: connection_settings }, name: :gnat_sup ) # Wait until the connection is actually established before issuing JS admin calls. wait_for_gnat = fn wait_for_gnat -> case Process.whereis(:gnat) do nil -> Process.sleep(100) wait_for_gnat.(wait_for_gnat) pid when is_pid(pid) -> :ok end end wait_for_gnat.(wait_for_gnat) Logger.info("connected to cluster") # Create (or reuse) a 3-replica stream + durable consumer. stream = %Stream{ name: stream_name, subjects: [subject], num_replicas: 3, storage: :file } case Stream.create(:gnat, stream) do {:ok, _} -> Logger.info("created stream #{stream_name}") {:error, %{"err_code" => 10058}} -> Logger.info("stream #{stream_name} already exists") {:error, other} -> Logger.warning("stream create returned: #{inspect(other)}") end consumer = %Consumer{ stream_name: stream_name, durable_name: consumer_name, ack_policy: :all } case Consumer.create(:gnat, consumer) do {:ok, _} -> Logger.info("created consumer #{consumer_name}") {:error, %{"err_code" => 10148}} -> Logger.info("consumer #{consumer_name} already exists") {:error, other} -> Logger.warning("consumer create returned: #{inspect(other)}") end # ---------- The PullConsumer under test ---------- defmodule FailoverConsumer do use Gnat.Jetstream.PullConsumer require Logger def start_link(opts), do: Gnat.Jetstream.PullConsumer.start_link(__MODULE__, opts) @impl true def init(opts) do {:ok, %{count: 0, batch_size: 10}, opts} end @impl true def handle_message(%{body: body, reply_to: reply_to}, state) do # reply_to carries delivery metadata: $JS.ACK....... meta = case reply_to do "$JS.ACK." <> rest -> rest other -> other end Logger.info("RECV ##{state.count + 1} body=#{inspect(body)} meta=#{meta}") {:ack, %{state | count: state.count + 1}} end @impl true def handle_status(%{status: status, description: desc}, state) do Logger.warning("STATUS #{status} #{desc}") {:ok, state} end def handle_status(%{status: status}, state) do Logger.warning("STATUS #{status}") {:ok, state} end @impl true def handle_connected(consumer_info, state) do Logger.info("connected, num_pending=#{consumer_info.num_pending}") {:ok, state} end end {:ok, _pc} = FailoverConsumer.start_link( connection_name: :gnat, stream_name: stream_name, consumer_name: consumer_name ) Logger.info("PullConsumer started; publishing one message per second") # ---------- Publisher loop ---------- # Uses Gnat.request so we wait for the JetStream publish ack. We only bump # the accepted counter when the server confirms the write — that way RECV # numbers track actual stream writes, not best-effort pub attempts. defmodule Publisher do require Logger def loop(subject, attempt \\ 1, accepted \\ 0) do body = "msg-#{accepted + 1}-#{System.system_time(:millisecond)}" result = try do Gnat.request(:gnat, subject, body, receive_timeout: 2_000) catch :exit, reason -> {:error, {:exit, reason}} end case result do {:ok, %{body: resp_body}} -> case Jason.decode(resp_body) do {:ok, %{"stream" => stream, "seq" => seq}} -> Logger.info("PUB ##{accepted + 1} ok stream=#{stream} seq=#{seq} body=#{body}") Process.sleep(1_000) loop(subject, attempt + 1, accepted + 1) {:ok, %{"error" => err}} -> Logger.warning( "PUB attempt ##{attempt} rejected by server: #{inspect(err)} body=#{body}" ) Process.sleep(1_000) loop(subject, attempt + 1, accepted) other -> Logger.warning("PUB attempt ##{attempt} unparsable ack: #{inspect(other)}") Process.sleep(1_000) loop(subject, attempt + 1, accepted) end {:error, reason} -> Logger.warning("PUB attempt ##{attempt} failed: #{inspect(reason)} body=#{body}") Process.sleep(1_000) loop(subject, attempt + 1, accepted) end end end Publisher.loop(subject) ================================================ FILE: scripts/cluster/n1.conf ================================================ port: 4223 http_port: 8223 server_name: n1 jetstream { store_dir: "./scripts/cluster/data/n1" } cluster { name: failover_test listen: 127.0.0.1:6223 routes: [ nats-route://127.0.0.1:6224 nats-route://127.0.0.1:6225 ] } ================================================ FILE: scripts/cluster/n2.conf ================================================ port: 4224 http_port: 8224 server_name: n2 jetstream { store_dir: "./scripts/cluster/data/n2" } cluster { name: failover_test listen: 127.0.0.1:6224 routes: [ nats-route://127.0.0.1:6223 nats-route://127.0.0.1:6225 ] } ================================================ FILE: scripts/cluster/n3.conf ================================================ port: 4225 http_port: 8225 server_name: n3 jetstream { store_dir: "./scripts/cluster/data/n3" } cluster { name: failover_test listen: 127.0.0.1:6225 routes: [ nats-route://127.0.0.1:6223 nats-route://127.0.0.1:6224 ] } ================================================ FILE: test/command_test.exs ================================================ defmodule Gnat.CommandTest do use ExUnit.Case, async: true alias Gnat.Command test "formatting a simple pub message" do command = Command.build(:pub, "topic", "payload", []) |> IO.iodata_to_binary() assert command == "PUB topic 7\r\npayload\r\n" end test "formatting a pub with reply_to set" do command = Command.build(:pub, "topic", "payload", reply_to: "INBOX") |> IO.iodata_to_binary() assert command == "PUB topic INBOX 7\r\npayload\r\n" end test "formatting a basic sub message" do command = Command.build(:sub, "foobar", 4, []) |> IO.iodata_to_binary() assert command == "SUB foobar 4\r\n" end test "formatting a sub with a queue group" do command = Command.build(:sub, "foobar", 5, queue_group: "us") |> IO.iodata_to_binary() assert command == "SUB foobar us 5\r\n" end test "formatting a simple unsub message" do command = Command.build(:unsub, 12, []) |> IO.iodata_to_binary() assert command == "UNSUB 12\r\n" end test "formatting an unsub message with max messages" do command = Command.build(:unsub, 12, max_messages: 3) |> IO.iodata_to_binary() assert command == "UNSUB 12 3\r\n" end end ================================================ FILE: test/fixtures/ca.pem ================================================ -----BEGIN CERTIFICATE----- MIIDXDCCAkQCCQDI2Vsry8+BDDANBgkqhkiG9w0BAQsFADBwMQswCQYDVQQGEwJV UzELMAkGA1UECAwCQ0ExEDAOBgNVBAoMB1N5bmFkaWExEDAOBgNVBAsMB25hdHMu aW8xEjAQBgNVBAMMCWxvY2FsaG9zdDEcMBoGCSqGSIb3DQEJARYNZGVyZWtAbmF0 cy5pbzAeFw0xOTEwMTcxMzAzNThaFw0yOTEwMTQxMzAzNThaMHAxCzAJBgNVBAYT AlVTMQswCQYDVQQIDAJDQTEQMA4GA1UECgwHU3luYWRpYTEQMA4GA1UECwwHbmF0 cy5pbzESMBAGA1UEAwwJbG9jYWxob3N0MRwwGgYJKoZIhvcNAQkBFg1kZXJla0Bu YXRzLmlvMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAohX2dXdHIDM5 yZDWk96b0mwRTHhBIOKtMPTTs/zKmlAgjjDxW7kSg0JimTNds9YbJ33FhcEJKXtV KH3Cn0uyZPS1VcTzPr7XP2QI+9SqqLuahkHAhgqoRwK62fTFJgzdZO0f9w9WwzMi gGk/v7KkKFa/9xKLCa9DTEJ9FA34HuYoBxXMZvypDm8d+0kxOCdThpzhKeucE4ya jFlvOP9/l7GyjlczzAD/nt/QhPfSeIx1MF0ICj5qzwPD/jB1ekoL9OShoHvoEyXo UO13GMdVmZqwJcS7Vk5XNEZoH0cxSw/SrZGCE9SFjR1t8TAe3QZiZ9E8EAg4IzJQ jfR2II5LiQIDAQABMA0GCSqGSIb3DQEBCwUAA4IBAQBIwib+0xLth/1+URtgQFn8 dvQNqnJjlqC27U48qiTCTC5vJWbQDqUg9o6gtwZyYEHQ7dMmn68ozDzcGTCxaikV n01Bj2ijODK96Jrm/P5aVkP5Cn06FfudluZI2Q/A1cqTsa8V4rj02PpwCcLEaDqX yhztlhbKypWrlGuWpVlDBWstyRar98vvRK1XEyBu2NHp2fy49cwJCub4Cmz920fh oiIwzXIKtfnf1GEjUnsuFPMgCxvhjirYNPWWjqaBldrM/dBJqwTyZf/p6g40vufN JJDc65c4tyRwBSBdFn+Q4zD44M0AR/8THAeIfsT42lyl8fMV5A4fe1nAVJDC4Z/H -----END CERTIFICATE----- ================================================ FILE: test/fixtures/client-cert.pem ================================================ -----BEGIN CERTIFICATE----- MIIDQjCCAiqgAwIBAgIJAJCSLX9jr5WzMA0GCSqGSIb3DQEBBQUAMHAxCzAJBgNV BAYTAlVTMQswCQYDVQQIDAJDQTEQMA4GA1UECgwHU3luYWRpYTEQMA4GA1UECwwH bmF0cy5pbzESMBAGA1UEAwwJbG9jYWxob3N0MRwwGgYJKoZIhvcNAQkBFg1kZXJl a0BuYXRzLmlvMB4XDTE5MTAxNzEzMjI0MloXDTI5MTAxNDEzMjI0MlowDTELMAkG A1UEBhMCVVMwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQCsnD6dO3oS VoV4yt+c/Ax+XvJPIjNGgThT16clj9fuFhPiZ0mI9pSZ8Kmm2/56F8nj3zFzcThw OpYemXtdB+Nj5Oi/mfc9XCf1tzcp2u6CgADUyNMbNg2L04qbjhKhTQzFIvhWO2oa ++k9CB4Tf1VuLmWTmpBUA20N5kTW98DX2OHHHsKbo26I8XxYCKKfE8xbuREsHSNv Oq5Hmg9qzuWANAnm4/12Ss9aGLucxcF0SWd3G7oohjGm/BKvSoUbc1v01kL/DBxJ 5zHyWioezYfLIv9wHEjtuuC+8Lye4NxZ26V0JVizYQT2MyhrByVgD3KTFmyfsK1K GPeeKR63YTQXAgMBAAGjQjBAMCkGA1UdEQQiMCCCCWxvY2FsaG9zdIcEfwAAAYEN ZGVyZWtAbmF0cy5pbzATBgNVHSUEDDAKBggrBgEFBQcDAjANBgkqhkiG9w0BAQUF AAOCAQEAfGUnzmpXXigAJxcnVKQy1ago+GGOAGldlDKIcHoofkYibhWWrrojulHF pRPRVKm2S/P4rRnSsjrPfpf6I2Icd+oVdVxrsWcN5itbul8Xymsjl2gMSJSHknYs wTYNjdM4opRioArK69aRa26xXlxRs8YpRErF8Nb5mkxgvtUgtM8t/T/28MBprc7x 7NuYvohKlOcWbgdBYI+e3CA2XLRG/A+9EmOe8g66vW/uY0eaiWduBJSwXhd+stjg elXYnK+EEUpJIK9DeS7r6k6HreNZ2FPM90RxdbMP7Q+i3bJwic4cJG3QOdLl+IqK tME8kUPD/63mEDHHMJjgAktgaFX4bQ== -----END CERTIFICATE----- ================================================ FILE: test/fixtures/client-key.pem ================================================ -----BEGIN PRIVATE KEY----- MIIEvwIBADANBgkqhkiG9w0BAQEFAASCBKkwggSlAgEAAoIBAQCsnD6dO3oSVoV4 yt+c/Ax+XvJPIjNGgThT16clj9fuFhPiZ0mI9pSZ8Kmm2/56F8nj3zFzcThwOpYe mXtdB+Nj5Oi/mfc9XCf1tzcp2u6CgADUyNMbNg2L04qbjhKhTQzFIvhWO2oa++k9 CB4Tf1VuLmWTmpBUA20N5kTW98DX2OHHHsKbo26I8XxYCKKfE8xbuREsHSNvOq5H mg9qzuWANAnm4/12Ss9aGLucxcF0SWd3G7oohjGm/BKvSoUbc1v01kL/DBxJ5zHy WioezYfLIv9wHEjtuuC+8Lye4NxZ26V0JVizYQT2MyhrByVgD3KTFmyfsK1KGPee KR63YTQXAgMBAAECggEBAKc6FHt2NPTxOAxn2C6aDmycBftesfiblnu8EWaVrmgu oYMV+CsmYZ+mhmZu+mNFCsam5JzoUvp/+BKbNeZSjx2nl0qRmvOqhdhLcbkuLybl ZmjAS64wNv2Bq+a6xRfaswWGtLuugkS0TCph4+mV0qmVb7mJ5ExQqWXu8kCl9QHn uKacp1wVFok9rmEI+byL1+Z01feKrkf/hcF6dk62U7zHNPajViJFTDww7hiHyfUH 6qsxIe1UWSNKtE61haEHkzqbDIDAy79jX4t3JobLToeVNCbJ7BSPf2IQSPJxELVL sidIJhndEjsbDR2CLpIF/EjsiSIaP7jh2zC9fxFpgSkCgYEA1qH0PH1JD5FqRV/p n9COYa6EifvSymGo4u/2FHgtX7wNSIQvqAVXenrQs41mz9E65womeqFXT/AZglaM 1PEjjwcFlDuLvUEYYJNgdXrIC515ZXS6TdvJ0JpQJLx28GzZ7h31tZXfwn68C3/i UGEHp+nN1BfBBQnsqvmGFFvHZFUCgYEAzeDlZHHijBlgHU+kGzKm7atJfAGsrv6/ tw7CIMEsL+z/y7pl3nwDLdZF+mLIvGuKlwIRajEzbYcEuVymCyG2/SmPMQEUf6j+ C1OmorX9CW8OwHmVCajkIgKn0ICFsF9iFv6aYZmm1kG48AIuYiQ7HOvY/MlilqFs 1p8sw6ZpQrsCgYEAj7Z9fQs+omfxymYAXnwc+hcKtAGkENL3bIzULryRVSrrkgTA jDaXbnFR0Qf7MWedkxnezfm+Js5TpkwhnGuiLaC8AZclaCFwGypTShZeYDifEmno XT2vkjfhNdfjo/Ser6vr3BxwaSDG9MQ6Wyu9HpeUtFD7c05D4++T8YnKpskCgYEA pCkcoIAStcWSFy0m3K0B3+dBvAiVyh/FfNDeyEFf24Mt4CPsEIBwBH+j4ugbyeoy YwC6JCPBLyeHA8q1d5DVmX4m+Fs1HioBD8UOzRUyA/CzIZSQ21f5OIlHiIDCmQUl cNJpBUQAfT2AmpgSphzfqcsBhWeLHjLvVx8rEYLC0fsCgYAiHdPZ3C0f7rWZP93N gY4DuldiO4d+KVsWAdBxeNgPznisUI7/ZZ/9NvCxGvA5NynyZr0qlpiKzVvtFJG8 1ZPUuFFRMAaWn9h5C+CwMPgk65tFC6lw/el0hpmcocSXVdiJEbkV0rnv9iGh0CYX HMACGrYlyZdDYM0CH/JAM+K/QQ== -----END PRIVATE KEY----- ================================================ FILE: test/fixtures/nkey_config ================================================ authorization: { users: [ { nkey: UBSUDO5PNPFR72YUCWWSN4ADPIEU3WESNZ35S3VZAWERPXCZSQTDH7SS } ] } ================================================ FILE: test/fixtures/nkey_seed ================================================ SUAMH3IDGSDQ2AVZFOWYAKNA7R2FXIZZSQ3BQMA5QNJRYP3ABIKDDP5DBA ================================================ FILE: test/fixtures/server-cert.pem ================================================ -----BEGIN CERTIFICATE----- MIIDPTCCAiWgAwIBAgIJAJCSLX9jr5W7MA0GCSqGSIb3DQEBBQUAMHAxCzAJBgNV BAYTAlVTMQswCQYDVQQIDAJDQTEQMA4GA1UECgwHU3luYWRpYTEQMA4GA1UECwwH bmF0cy5pbzESMBAGA1UEAwwJbG9jYWxob3N0MRwwGgYJKoZIhvcNAQkBFg1kZXJl a0BuYXRzLmlvMB4XDTE5MTAxNzEzNTcyNloXDTI5MTAxNDEzNTcyNlowDTELMAkG A1UEBhMCVVMwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQDm+0dlzcmi La+LzdVqeVQ8B1/rWnErK+VvvjH7FmVodg5Z5+RXyojpd9ZBrVd6QrLSVMQPfFvB vGGX4yI6Ph5KXUefa31vNOOMhp2FGSmaEVhETKGQ0xRh4VfaAerOP5Cunl0TbSyJ yjkVa7aeMtcqTEiFL7Ae2EtiMhTrMrYpBDQ8rzm2i1IyTb9DX5v7DUOmrSynQSlV yXCztRVGNL/kHlItpEku1SHt/AD3ogu8EgqQZFB8xRRw9fubYgh4Q0kx80e4k9Qt TKncF3B2NGb/ZcE5Z+mmHIBq8J2zKMijOrdd3m5TbQmzDbETEOjs4L1eoZRLcL/c vYu5gmXdr4F7AgMBAAGjPTA7MBoGA1UdEQQTMBGCCWxvY2FsaG9zdIcEfwAAATAd BgNVHSUEFjAUBggrBgEFBQcDAgYIKwYBBQUHAwEwDQYJKoZIhvcNAQEFBQADggEB ADQYaEjWlOb9YzUnFGjfDC06dRZjRmK8TW/4GiDHIDk5TyZ1ROtskvyhVyTZJ5Vs qXOKJwpps0jK2edtrvZ7xIGw+Y41oPgYYhr5TK2c+oi2UOHG4BXqRbuwz/5cU+nM ZWOG1OrHBCbrMSeFsn7rzETnd8SZnw6ZE7LI62WstdoCY0lvNfjNv3kY/6hpPm+9 0bVzurZ28pdJ6YEJYgbPcOvxSzGDXTw9LaKEmqknTsrBKI2qm+myVTbRTimojYTo rw/xjHESAue/HkpOwWnFTOiTT+V4hZnDXygiSy+LWKP4zLnYOtsn0lN9OmD0z+aa gpoVMSncu2jMIDZX63IkQII= -----END CERTIFICATE----- ================================================ FILE: test/fixtures/server-key.pem ================================================ -----BEGIN PRIVATE KEY----- MIIEuwIBADANBgkqhkiG9w0BAQEFAASCBKUwggShAgEAAoIBAQDm+0dlzcmiLa+L zdVqeVQ8B1/rWnErK+VvvjH7FmVodg5Z5+RXyojpd9ZBrVd6QrLSVMQPfFvBvGGX 4yI6Ph5KXUefa31vNOOMhp2FGSmaEVhETKGQ0xRh4VfaAerOP5Cunl0TbSyJyjkV a7aeMtcqTEiFL7Ae2EtiMhTrMrYpBDQ8rzm2i1IyTb9DX5v7DUOmrSynQSlVyXCz tRVGNL/kHlItpEku1SHt/AD3ogu8EgqQZFB8xRRw9fubYgh4Q0kx80e4k9QtTKnc F3B2NGb/ZcE5Z+mmHIBq8J2zKMijOrdd3m5TbQmzDbETEOjs4L1eoZRLcL/cvYu5 gmXdr4F7AgMBAAECggEBAK4sr3MiEbjcsHJAvXyzjwRRH1Bu+8VtLW7swe2vvrpd w4aiKXrV/BXpSsRtvPgxkXyvdMSkpuBZeFI7cVTwAJFc86RQPt77x9bwr5ltFwTZ rXCbRH3b3ZPNhByds3zhS+2Q92itu5cPyanQdn2mor9/lHPyOOGZgobCcynELL6R wRElkeDyf5ODuWEd7ADC5IFyZuwb3azNVexIK+0yqnMmv+QzEW3hsycFmFGAeB7v MIMjb2BhLrRr6Y5Nh+k58yM5DCf9h/OJhDpeXwLkxyK4BFg+aZffEbUX0wHDMR7f /nMv1g6cKvDWiLU8xLzez4t2qNIBNdxw5ZSLyQRRolECgYEA+ySTKrBAqI0Uwn8H sUFH95WhWUXryeRyGyQsnWAjZGF1+d67sSY2un2W6gfZrxRgiNLWEFq9AaUs0MuH 6syF4Xwx/aZgU/gvsGtkgzuKw1bgvekT9pS/+opmHRCZyQAFEHj0IEpzyB6rW1u/ LdlR3ShEENnmXilFv/uF/uXP5tMCgYEA63LiT0w46aGPA/E+aLRWU10c1eZ7KdhR c3En6zfgIxgFs8J38oLdkOR0CF6T53DSuvGR/OprVKdlnUhhDxBgT1oQjK2GlhPx JV5uMvarJDJxAwsF+7T4H2QtZ00BtEfpyp790+TlypSG1jo/BnSMmX2uEbV722lY hzINLY49obkCgYBEpN2YyG4T4+PtuXznxRkfogVk+kiVeVx68KtFJLbnw//UGT4i EHjbBmLOevDT+vTb0QzzkWmh3nzeYRM4aUiatjCPzP79VJPsW54whIDMHZ32KpPr TQMgPt3kSdpO5zN7KiRIAzGcXE2n/e7GYGUQ1uWr2XMu/4byD5SzdCscQwJ/Ymii LoKtRvk/zWYHr7uwWSeR5dVvpQ3E/XtONAImrIRd3cRqXfJUqTrTRKxDJXkCmyBc 5FkWg0t0LUkTSDiQCJqcUDA3EINFR1kwthxja72pfpwc5Be/nV9BmuuUysVD8myB qw8A/KsXsHKn5QrRuVXOa5hvLEXbuqYw29mX6QKBgDGDzIzpR9uPtBCqzWJmc+IJ z4m/1NFlEz0N0QNwZ/TlhyT60ytJNcmW8qkgOSTHG7RDueEIzjQ8LKJYH7kXjfcF 6AJczUG5PQo9cdJKo9JP3e1037P/58JpLcLe8xxQ4ce03zZpzhsxR2G/tz8DstJs b8jpnLyqfGrcV2feUtIZ -----END PRIVATE KEY----- ================================================ FILE: test/gnat/consumer_supervisor_test.exs ================================================ defmodule Gnat.ConsumerSupervisorTest do alias Gnat.ConsumerSupervisor use ExUnit.Case, async: true # these requests are being handled by `ExampleServer` in the `test_helper.exs` file test "successful requests work fine" do assert {:ok, %{body: "Re: hi"}} = Gnat.request(:test_connection, "example.good", "hi") end test "catches returned errors" do assert {:ok, %{body: "400 error"}} = Gnat.request(:test_connection, "example.error", "hi") end test "catches raised errors" do assert {:ok, %{body: "500 error"}} = Gnat.request(:test_connection, "example.raise", "hi") end # The happy path is setup in `test_helper.exs` # check the ExampleService module for the implementation of the endpoints test "microservice endpoint add works" do assert {:ok, %{body: "6"}} = Gnat.request(:test_connection, "calc.add", "foo") end test "microservice endpoint sub works" do assert {:ok, %{body: "4"}} = Gnat.request(:test_connection, "calc.sub", "foo") end test "microservice endpoint errors properly" do assert {:ok, %{body: "500 error"}} = Gnat.request(:test_connection, "calc.sub", "error") end test "service endpoint ping response" do {:ok, %{body: body}} = Gnat.request(:test_connection, "$SRV.PING.exampleservice", "") payload = Jason.decode!(body, keys: :atoms) assert payload.name == "exampleservice" assert is_binary(payload.id) assert payload.version == "0.1.0" assert payload.metadata == %{} end test "services info response" do {:ok, %{body: body}} = Gnat.request(:test_connection, "$SRV.INFO.exampleservice", "") payload = Jason.decode!(body, keys: :atoms) assert payload.name == "exampleservice" assert is_binary(payload.id) assert payload.version == "0.1.0" assert payload.description == "This is an example service" assert payload.metadata == %{} [add, sub] = Enum.sort_by(payload.endpoints, & &1.name) assert add == %{ name: "add", subject: "calc.add", queue_group: "q", metadata: %{} } assert sub == %{ name: "sub", subject: "calc.sub", queue_group: "q", metadata: %{} } end test "service endpoint stats response" do # at least 1 error, at least 1 request, non-zero processing time assert {:ok, %{body: "4"}} = Gnat.request(:test_connection, "calc.sub", "foo") assert {:ok, %{body: "6"}} = Gnat.request(:test_connection, "calc.add", "foo") assert {:ok, %{body: "500 error"}} = Gnat.request(:test_connection, "calc.sub", "error") {:ok, %{body: body}} = Gnat.request(:test_connection, "$SRV.STATS.exampleservice", "") payload = Jason.decode!(body, keys: :atoms) assert Enum.at(payload.endpoints, 0) |> Map.get(:processing_time) > 1000 assert Enum.at(payload.endpoints, 0) |> Map.get(:num_requests) > 0 assert Enum.at(payload.endpoints, 1) |> Map.get(:processing_time) > 1000 assert Enum.at(payload.endpoints, 1) |> Map.get(:num_requests) > 0 end test "validates the version of a service" do bad = %{ name: "exampleservice", description: "This is an example service", version: "0.1", endpoints: [ %{ name: "add", group_name: "calc" } ] } assert {:error, message} = start_service_supervisor(bad) assert message =~ "Version '0.1' does not conform to semver specification" end test "validates the name of a service" do bad = %{ name: "example!", description: "This is an example service", version: "0.1.0", endpoints: [ %{ name: "add", group_name: "calc" } ] } assert {:error, message} = start_service_supervisor(bad) assert message =~ "Service name 'example!' is invalid." end test "validates the name of an endpoint" do bad = %{ name: "example", description: "This is an example service", version: "0.1.0", endpoints: [ %{ name: "add some stuff", group_name: "calc" } ] } assert {:error, message} = start_service_supervisor(bad) assert message =~ "Endpoint name 'add some stuff' is not valid" end test "validates metadata of a service" do bad = %{ name: "exampleservice", description: "This is an example service", version: "0.0.1", endpoints: [ %{ name: "add", group_name: "calc", metadata: %{:blarg => :thisisbad} } ] } assert {:error, message} = start_service_supervisor(bad) assert message =~ "At least one key or value found in metadata that was not a string" end # In OTP 26 the GenServer.init behavior changed to do a process EXIT when returning a # {:stop, error} in GenServer.init # We inherit this behavior, so for the purpose of testing, we trap those process exits # to make sure we can process the `{:error, error}` tuple before the process exit kills # our ExUnit test defp start_service_supervisor(service_config) do Process.flag(:trap_exit, true) config = %{ connection_name: :something, module: ExampleService, service_definition: service_config } ConsumerSupervisor.start_link(config) end end ================================================ FILE: test/gnat/handshake_test.exs ================================================ defmodule Gnat.HandshakeTest do use ExUnit.Case, async: true alias Gnat.Handshake describe "negotiate_settings/2" do test "respects server auth_required setting" do server_settings = %{auth_required: true} user_settings = %{username: "test", password: "secret"} result = Handshake.negotiate_settings(server_settings, user_settings) assert result[:user] == "test" assert result[:pass] == "secret" end test "allows client to force auth when server doesn't require it" do server_settings = %{} user_settings = %{username: "test", password: "secret", auth_required: true} result = Handshake.negotiate_settings(server_settings, user_settings) assert result[:user] == "test" assert result[:pass] == "secret" end test "allows client to force auth with token when server doesn't require it" do server_settings = %{} user_settings = %{token: "my-secret-token", auth_required: true} result = Handshake.negotiate_settings(server_settings, user_settings) assert result[:auth_token] == "my-secret-token" end test "doesn't send auth when neither server nor client requires it" do server_settings = %{} user_settings = %{username: "test", password: "secret"} result = Handshake.negotiate_settings(server_settings, user_settings) refute Map.has_key?(result, :user) refute Map.has_key?(result, :pass) refute Map.has_key?(result, :auth_token) end test "client auth_required setting takes precedence over server setting being false" do server_settings = %{auth_required: false} user_settings = %{username: "test", password: "secret", auth_required: true} result = Handshake.negotiate_settings(server_settings, user_settings) assert result[:user] == "test" assert result[:pass] == "secret" end test "works with nkey authentication when client forces auth" do nonce = "test-nonce-value" server_settings = %{nonce: nonce} user_settings = %{ nkey_seed: "SUAIBDPBAUTWCWBKIO6XHQNINK5FWJW4OHLXC3HQ2KFE4PEJUA44CNHTC4", auth_required: true } result = Handshake.negotiate_settings(server_settings, user_settings) assert Map.has_key?(result, :sig) assert Map.has_key?(result, :nkey) assert result[:protocol] == 1 end test "works with JWT+nkey authentication when client forces auth" do nonce = "test-nonce-value" server_settings = %{nonce: nonce} user_settings = %{ nkey_seed: "SUAIBDPBAUTWCWBKIO6XHQNINK5FWJW4OHLXC3HQ2KFE4PEJUA44CNHTC4", jwt: "eyJ0eXAiOiJKV1QiLCJhbGciOiJlZDI1NTE5LW5rZXkifQ.eyJqdGkiOiJPUkhQUERHQ1FHRVdPSkZOUVIzM0tFSzVYT0lHSElNNlFOTVFOUUVIVlJLWVpGUkQ3NFNBIiwiaWF0IjoxNjM4MzMzMjI4LCJpc3MiOiJBQlpMM1pSRkdYNTQzRkU1SkdDMkVFQkJRVVhSREQ1TFdWN1dYSEdCSEdOUko2Nks0VUNJUEFHMyIsIm5hbWUiOiJ0ZXN0LXVzZXIiLCJzdWIiOiJVQzJGRllPUTVQWUEyQU5aREFCV1daSEhNRE5JVVdLQ0VITldNSUNCNlo2U1hLNEdOVUFZUUdCUCIsIm5hdHMiOnsicHViIjp7fSwic3ViIjp7fSwic3VicyI6LTEsImRhdGEiOi0xLCJwYXlsb2FkIjotMSwidHlwZSI6InVzZXIiLCJ2ZXJzaW9uIjoyfX0.test-signature", auth_required: true } result = Handshake.negotiate_settings(server_settings, user_settings) assert Map.has_key?(result, :sig) assert Map.has_key?(result, :jwt) assert result[:protocol] == 1 end end end ================================================ FILE: test/gnat/parsec_property_test.exs ================================================ defmodule Gnat.ParsecPropertyTest do use ExUnit.Case, async: true use PropCheck import Gnat.Generators, only: [message: 0, protocol_message: 0] alias Gnat.Parsec @numtests (System.get_env("N") || "100") |> String.to_integer() @tag :property property "can parse any message" do numtests( @numtests * 10, forall map <- message() do {_parser, [parsed]} = Parsec.new() |> Parsec.parse(map.binary) {:msg, parsed_subject, parsed_sid, parsed_reply_to, parsed_payload} = parsed parsed_payload == map.payload && parsed_subject == map.subject && parsed_sid == map.sid && parsed_reply_to == map.reply_to end ) end @tag :property property "can parse messages split into arbitrary chunks" do numtests( @numtests, forall messages <- list(message()) do frames = messages |> Enum.reduce("", fn %{binary: b}, acc -> acc <> b end) |> random_chunk payloads = Enum.map(messages, fn msg -> msg.payload end) {parser, parsed} = Enum.reduce(frames, {Parsec.new(), []}, fn frame, {parser, acc} -> {parser, parsed_messages} = Parsec.parse(parser, frame) payloads = Enum.map(parsed_messages, &elem(&1, 4)) {parser, acc ++ payloads} end) parser.partial == nil && parsed == payloads end ) end @tag :property property "can parse any sequence of protocol messages without exception" do numtests( @numtests, forall messages <- list(protocol_message()) do parser = Enum.reduce(messages, Parsec.new(), fn %{binary: bin}, parser -> {parser, [_parsed]} = Parsec.parse(parser, bin) parser end) parser.partial == nil end ) end @tag :property property "can parse any sequence of protocol messages, randomly chunked without exception" do numtests( @numtests, forall messages <- list(protocol_message()) do bin = Enum.reduce(messages, "", fn %{binary: part}, acc -> acc <> part end) chunks = random_chunk(bin) {parser, parsed} = Enum.reduce(chunks, {Parsec.new(), []}, fn bin, {parser, acc} -> {parser, parsed} = Parsec.parse(parser, bin) {parser, acc ++ parsed} end) parser.partial == nil && Enum.count(parsed) == Enum.count(messages) end ) end def random_chunk(binary), do: random_chunk(binary, []) def random_chunk("", list), do: Enum.reverse(list) def random_chunk(binary, list) do size = :rand.uniform(128) remaining = byte_size(binary) case remaining <= size do true -> random_chunk("", [binary | list]) false -> random_chunk(:binary.part(binary, size, remaining - size), [ :binary.part(binary, 0, size) | list ]) end end end ================================================ FILE: test/gnat/parsec_test.exs ================================================ defmodule Gnat.ParsecTest do use ExUnit.Case, async: true alias Gnat.Parsec test "parsing a complete message" do {parser_state, [parsed_message]} = Parsec.new() |> Parsec.parse("MSG topic 13 4\r\ntest\r\n") assert parser_state.partial == nil assert parsed_message == {:msg, "topic", 13, nil, "test"} end test "parsing a complete message with newlines in it" do {parser_state, [parsed_message]} = Parsec.new() |> Parsec.parse("MSG topic 13 10\r\ntest\r\nline\r\n") assert parser_state.partial == nil assert parsed_message == {:msg, "topic", 13, nil, "test\r\nline"} end test "parsing multiple messages" do {parser_state, [msg1, msg2]} = Parsec.new() |> Parsec.parse("MSG t1 1 3\r\nwat\r\nMSG t2 2 4\r\ndawg\r\n") assert parser_state.partial == nil assert msg1 == {:msg, "t1", 1, nil, "wat"} assert msg2 == {:msg, "t2", 2, nil, "dawg"} end test "parsing a message with a reply to" do {parser_state, [parsed_message]} = Parsec.new() |> Parsec.parse("MSG topic 13 me 10\r\ntest\r\nline\r\n") assert parser_state.partial == nil assert parsed_message == {:msg, "topic", 13, "me", "test\r\nline"} end test "handling _INBOX subjects" do inbox = "_INBOX.Rf+MI+V1+9pUCgC+.BChhlI06WHyCTYor" {parser_state, [parsed_message]} = Parsec.new() |> Parsec.parse("MSG topic 13 #{inbox} 10\r\ntest\r\nline\r\n") assert parser_state.partial == nil assert parsed_message == {:msg, "topic", 13, inbox, "test\r\nline"} end test "parsing messages with headers - single header no reply" do binary = "HMSG SUBJECT 1 23 30\r\nNATS/1.0\r\nHeader: X\r\n\r\nPAYLOAD\r\n" {state, [parsed]} = Parsec.new() |> Parsec.parse(binary) assert state.partial == nil assert parsed == {:hmsg, "SUBJECT", 1, nil, nil, nil, [{"header", "X"}], "PAYLOAD"} end test "parsing messages with headers - single header" do binary = "HMSG SUBJECT 1 REPLY 23 30\r\nNATS/1.0\r\nHeader: X\r\n\r\nPAYLOAD\r\n" {state, [parsed]} = Parsec.new() |> Parsec.parse(binary) assert state.partial == nil assert parsed == {:hmsg, "SUBJECT", 1, "REPLY", nil, nil, [{"header", "X"}], "PAYLOAD"} end # This example comes from https://github.com/nats-io/nats-architecture-and-design/blob/cb8f68af6ba730c00a6aa174dedaa217edd9edc6/adr/ADR-9.md test "parsing idle heartbeat messages" do binary = "HMSG my.messages 2 75 75\r\nNATS/1.0 100 Idle Heartbeat\r\nNats-Last-Consumer: 0\r\nNats-Last-Stream: 0\r\n\r\n\r\n" {state, [parsed]} = Parsec.new() |> Parsec.parse(binary) assert state.partial == nil assert parsed == {:hmsg, "my.messages", 2, nil, "100", "Idle Heartbeat", [{"nats-last-consumer", "0"}, {"nats-last-stream", "0"}], ""} end # This example comes from https://github.com/nats-io/nats-architecture-and-design/blob/682d5cd5f21d18502da70025727128a407655250/adr/ADR-13.md test "parsing no wait pull request responses" do binary = "HMSG _INBOX.x7tkDPDLCOEknrfB4RH1V7.OgY4M7 2 28 28\r\nNATS/1.0 404 No Messages\r\n\r\n\r\n" {state, [parsed]} = Parsec.new() |> Parsec.parse(binary) assert state.partial == nil assert parsed == {:hmsg, "_INBOX.x7tkDPDLCOEknrfB4RH1V7.OgY4M7", 2, nil, "404", "No Messages", [], ""} end test "parsing no_responders messages" do binary = "HMSG _INBOX.10ahfXw89Nx5htVf.7Il73yuah/RHW6w8 0 16 16\r\nNATS/1.0 503\r\n\r\n\r\n" {state, [parsed]} = Parsec.new() |> Parsec.parse(binary) assert state.partial == nil assert parsed == {:hmsg, "_INBOX.10ahfXw89Nx5htVf.7Il73yuah/RHW6w8", 0, nil, "503", nil, [], ""} end test "parsing PING message" do {parser_state, [parsed_message]} = Parsec.new() |> Parsec.parse("PING\r\n") assert parser_state.partial == nil assert parsed_message == :ping end test "parsing a complete message with case insensitive command" do {parser_state, [parsed_message]} = Parsec.new() |> Parsec.parse("msg topic 13 4\r\ntest\r\n") assert parser_state.partial == nil assert parsed_message == {:msg, "topic", 13, nil, "test"} end test "parsing case insensitive ping message" do {parser_state, [parsed_message]} = Parsec.new() |> Parsec.parse("ping\r\n") assert parser_state.partial == nil assert parsed_message == :ping end test "parsing partial messages" do {parser_state, [parsed_message]} = Parsec.new() |> Parsec.parse("PING\r\nMSG topic 11 4\r\nOH") assert parsed_message == :ping assert parser_state.partial == "MSG topic 11 4\r\nOH" {parser_state, [msg1, msg2]} = Parsec.parse(parser_state, "AI\r\nMSG topic 11 3\r\nWAT\r\nMSG topic") assert msg1 == {:msg, "topic", 11, nil, "OHAI"} assert msg2 == {:msg, "topic", 11, nil, "WAT"} assert parser_state.partial == "MSG topic" end test "parsing INFO message" do {parser_state, [parsed_message]} = Parsec.new() |> Parsec.parse( "INFO {\"server_id\":\"1ec445b504f4edfb4cf7927c707dd717\",\"version\":\"0.6.6\",\"go\":\"go1.4.2\",\"host\":\"0.0.0.0\",\"port\":4222,\"auth_required\":false,\"ssl_required\":false,\"max_payload\":1048576}\r\n" ) assert parser_state.partial == nil assert parsed_message == {:info, %{ server_id: "1ec445b504f4edfb4cf7927c707dd717", version: "0.6.6", go: "go1.4.2", host: "0.0.0.0", port: 4222, auth_required: false, ssl_required: false, max_payload: 1_048_576 }} end test "parsing PONG message" do {parser_state, [parsed_message]} = Parsec.new() |> Parsec.parse("PONG\r\n") assert parser_state.partial == nil assert parsed_message == :pong end test "parsing -ERR message" do {parser_state, [parsed_message]} = Parsec.new() |> Parsec.parse("-ERR 'Unknown Protocol Operation'\r\n") assert parser_state.partial == nil assert parsed_message == {:error, "Unknown Protocol Operation"} end test "parsing -ERR messages in the middle of other traffic" do assert {parser, [:ping]} = Parsec.new() |> Parsec.parse("PING\r\n-ERR 'Unknown Pro") assert {_, [{:error, "Unknown Protocol Operation"}]} = parser |> Parsec.parse("tocol Operation'\r\nMSG") end end ================================================ FILE: test/gnat_property_test.exs ================================================ defmodule GnatPropertyTest do use ExUnit.Case, async: true use PropCheck import Gnat.Generators, only: [message: 0] @numtests (System.get_env("N") || "100") |> String.to_integer() @tag :property property "can publish to random subjects" do numtests( @numtests * 2, forall %{subject: subject, payload: payload} <- message() do Gnat.pub(:test_connection, subject, payload) == :ok end ) end @tag :property property "can subscribe, publish, receive and unsubscribe from subjects" do numtests( @numtests * 2, forall %{subject: subject, payload: payload} <- message() do {:ok, ref} = Gnat.sub(:test_connection, self(), subject) :ok = Gnat.pub(:test_connection, subject, payload) assert_receive {:msg, %{topic: ^subject, body: ^payload, reply_to: nil}}, 500 Gnat.unsub(:test_connection, ref) == :ok end ) end @tag :property property "can make requests to an echo endpoint" do numtests( @numtests * 2, forall %{subject: subject, payload: payload} <- message() do {:ok, msg} = Gnat.request(:test_connection, "rpc.#{subject}", payload) msg.body == payload end ) end end ================================================ FILE: test/gnat_test.exs ================================================ defmodule GnatTest do use ExUnit.Case, async: true doctest Gnat setup context do CheckForExpectedNatsServers.check(Map.keys(context)) :ok end test "connect to a server" do {:ok, pid} = Gnat.start_link() assert Process.alive?(pid) :ok = Gnat.stop(pid) end # We have to skip this test in CI builds because CircleCI doesn't enable IPv6 in it's docker # configuration. See https://circleci.com/docs/faq#can-i-use-ipv6-in-my-tests @tag :ci_skip test "connect to a server over IPv6" do {:ok, pid} = Gnat.start_link(%{host: ~c"::1", tcp_opts: [:binary, :inet6]}) assert Process.alive?(pid) :ok = Gnat.stop(pid) end @tag :multi_server test "connect to a server with user/pass authentication" do connection_settings = %{ host: "localhost", port: 4223, tcp_opts: [:binary], username: "bob", password: "alice" } {:ok, pid} = Gnat.start_link(connection_settings) assert Process.alive?(pid) :ok = Gnat.stop(pid) end @tag :multi_server test "connect to a server with token authentication" do connection_settings = %{ host: "localhost", port: 4226, tcp_opts: [:binary], token: "SpecialToken", auth_required: true } {:ok, pid} = Gnat.start_link(connection_settings) assert Process.alive?(pid) :ok = Gnat.stop(pid) end @tag :multi_server test "connet to a server which requires TLS" do connection_settings = %{port: 4224, tls: true} {:ok, gnat} = Gnat.start_link(connection_settings) assert Gnat.stop(gnat) == :ok end @tag :multi_server test "connect to a server which requires TLS with a client certificate" do connection_settings = %{ port: 4225, tls: true, ssl_opts: [ certfile: "test/fixtures/client-cert.pem", keyfile: "test/fixtures/client-key.pem" ] } {:ok, gnat} = Gnat.start_link(connection_settings) assert Gnat.stop(gnat) == :ok end @tag :multi_server test "connect to a server which requires nkeys" do connection_settings = %{ port: 4227, nkey_seed: File.read!("test/fixtures/nkey_seed") } {:ok, gnat} = Gnat.start_link(connection_settings) assert Gnat.stop(gnat) == :ok end test "subscribe to topic and receive a message" do {:ok, pid} = Gnat.start_link() {:ok, _ref} = Gnat.sub(pid, self(), "test") :ok = Gnat.pub(pid, "test", "yo dawg") assert_receive {:msg, %{topic: "test", body: "yo dawg", reply_to: nil}}, 1000 :ok = Gnat.stop(pid) end test "subscribe to topic and receive a message with headers" do {:ok, pid} = Gnat.start_link() {:ok, _ref} = Gnat.sub(pid, self(), "sub_with_headers") headers = [{"X", "foo"}] :ok = Gnat.pub(pid, "sub_with_headers", "yo dawg", headers: headers) assert_receive {:msg, %{ topic: "sub_with_headers", body: "yo dawg", reply_to: nil, headers: [{"x", "foo"}] }}, 1000 :ok = Gnat.stop(pid) end test "subscribe receive a message with a reply_to" do {:ok, pid} = Gnat.start_link() {:ok, _ref} = Gnat.sub(pid, self(), "with_reply") :ok = Gnat.pub(pid, "with_reply", "yo dawg", reply_to: "me") assert_receive {:msg, %{topic: "with_reply", reply_to: "me", body: "yo dawg"}}, 1000 :ok = Gnat.stop(pid) end test "subscribe receive a message with a reply_to and headers" do {:ok, pid} = Gnat.start_link() {:ok, _ref} = Gnat.sub(pid, self(), "with_reply") headers = [{"x", "y"}] :ok = Gnat.pub(pid, "with_reply", "yo dawg", reply_to: "me", headers: headers) assert_receive {:msg, %{topic: "with_reply", reply_to: "me", body: "yo dawg", headers: ^headers}}, 1000 :ok = Gnat.stop(pid) end test "receive multiple messages" do {:ok, pid} = Gnat.start_link() {:ok, _ref} = Gnat.sub(pid, self(), "test") :ok = Gnat.pub(pid, "test", "message 1") :ok = Gnat.pub(pid, "test", "message 2") :ok = Gnat.pub(pid, "test", "message 3") assert_receive {:msg, %{topic: "test", body: "message 1", reply_to: nil}}, 500 assert_receive {:msg, %{topic: "test", body: "message 2", reply_to: nil}}, 500 assert_receive {:msg, %{topic: "test", body: "message 3", reply_to: nil}}, 500 :ok = Gnat.stop(pid) end test "subscribing to the same topic multiple times" do {:ok, pid} = Gnat.start_link() {:ok, _sub1} = Gnat.sub(pid, self(), "dup") {:ok, _sub2} = Gnat.sub(pid, self(), "dup") :ok = Gnat.pub(pid, "dup", "yo") :ok = Gnat.pub(pid, "dup", "ma") assert_receive {:msg, %{topic: "dup", body: "yo"}}, 500 assert_receive {:msg, %{topic: "dup", body: "yo"}}, 500 assert_receive {:msg, %{topic: "dup", body: "ma"}}, 500 assert_receive {:msg, %{topic: "dup", body: "ma"}}, 500 end test "subscribing to the same topic multiple times with a queue group" do {:ok, pid} = Gnat.start_link() {:ok, _sub1} = Gnat.sub(pid, self(), "dup", queue_group: "us") {:ok, _sub2} = Gnat.sub(pid, self(), "dup", queue_group: "us") :ok = Gnat.pub(pid, "dup", "yo") :ok = Gnat.pub(pid, "dup", "ma") assert_receive {:msg, %{topic: "dup", body: "yo"}}, 500 assert_receive {:msg, %{topic: "dup", body: "ma"}}, 500 receive do {:msg, %{topic: _topic}} = msg -> flunk("Received duplicate message: #{inspect(msg)}") after 200 -> :ok end end test "unsubscribing from a topic" do topic = "testunsub" {:ok, pid} = Gnat.start_link() {:ok, sub_ref} = Gnat.sub(pid, self(), topic) :ok = Gnat.pub(pid, topic, "msg1") assert_receive {:msg, %{topic: ^topic, body: "msg1"}}, 500 :ok = Gnat.unsub(pid, sub_ref) :ok = Gnat.pub(pid, topic, "msg2") receive do {:msg, %{topic: _topic, body: _body}} = msg -> flunk("Received message after unsubscribe: #{inspect(msg)}") after 200 -> :ok end end test "unsubscribing from a topic after a maximum number of messages" do topic = "testunsub_maxmsg" {:ok, pid} = Gnat.start_link() {:ok, sub_ref} = Gnat.sub(pid, self(), topic) :ok = Gnat.unsub(pid, sub_ref, max_messages: 2) :ok = Gnat.pub(pid, topic, "msg1") :ok = Gnat.pub(pid, topic, "msg2") :ok = Gnat.pub(pid, topic, "msg3") assert_receive {:msg, %{topic: ^topic, body: "msg1"}}, 500 assert_receive {:msg, %{topic: ^topic, body: "msg2"}}, 500 receive do {:msg, _topic, _msg} = msg -> flunk("Received message after unsubscribe: #{inspect(msg)}") after 200 -> :ok end end test "subscription is cleaned up when the subscribing process dies" do topic = "testcleanup" test_pid = self() {:ok, pid} = Gnat.start_link() # one subscription created at boot assert {:ok, 1} = Gnat.active_subscriptions(pid) %Task{pid: task_pid} = Task.async(fn -> Gnat.sub(pid, self(), topic) assert {:ok, 2} = Gnat.active_subscriptions(pid) send(test_pid, "subscribed") receive do :done -> :ok end end) assert_receive "subscribed", 1_000 Gnat.server_info(pid) Process.monitor(task_pid) send(task_pid, :done) assert_receive {:DOWN, _ref, :process, ^task_pid, _reason}, 1_000 assert {:ok, 1} = Gnat.active_subscriptions(pid) end test "request-reply convenience function" do topic = "req-resp" {:ok, pid} = Gnat.start_link() spin_up_echo_server_on_topic(self(), pid, topic) # Wait for server to spawn and subscribe. assert_receive(true, 100) {:ok, msg} = Gnat.request(pid, topic, "ohai", receive_timeout: 500) assert msg.body == "ohai" end test "request-reply convenience function with headers" do topic = "req-resp" {:ok, pid} = Gnat.start_link() spin_up_echo_server_on_topic(self(), pid, topic) # Wait for server to spawn and subscribe. assert_receive(true, 100) headers = [{"accept", "json"}] {:ok, msg} = Gnat.request(pid, topic, "ohai", receive_timeout: 500, headers: headers) assert msg.body == "ohai" assert msg.headers == headers end @tag timeout: 100 test "request-reply no_responders" do topic = "nobody-is-listening-to-this-topic" {:ok, pid} = Gnat.start_link(%{no_responders: true}) assert {:error, :no_responders} = Gnat.request(pid, topic, "ohai") end @tag timeout: 100 test "request-reply timeout" do topic = "nobody-is-listening-to-this-topic" {:ok, pid} = Gnat.start_link() assert {:error, :timeout} = Gnat.request(pid, topic, "ohai", receive_timeout: 5) end test "request_multi convenience function with no maximum messages" do topic = "req.multi" {:ok, pid} = Gnat.start_link() # start 4 servers to get 4 responses Enum.each(1..4, fn _i -> spin_up_echo_server_on_topic(self(), pid, topic) assert_receive(true, 100) end) {:ok, messages} = Gnat.request_multi(pid, topic, "ohai", receive_timeout: 500) assert Enum.count(messages) == 4 assert Enum.all?(messages, fn msg -> msg.body == "ohai" end) end test "request_multi convenience function with maximum messages" do topic = "req.multi2" {:ok, pid} = Gnat.start_link() # start 4 servers to get 4 responses Enum.each(1..4, fn _i -> spin_up_echo_server_on_topic(self(), pid, topic) assert_receive(true, 100) end) {:ok, messages} = Gnat.request_multi(pid, topic, "ohai", max_messages: 4, receive_timeout: 500) assert Enum.count(messages) == 4 assert Enum.all?(messages, fn msg -> msg.body == "ohai" end) end test "request_multi convenience function with maximum messages not met" do topic = "req.multi2" {:ok, pid} = Gnat.start_link() # start 4 servers to get 4 responses Enum.each(1..4, fn _i -> spin_up_echo_server_on_topic(self(), pid, topic) assert_receive(true, 100) end) {:ok, messages} = Gnat.request_multi(pid, topic, "ohai", max_messages: 8, receive_timeout: 500) assert Enum.count(messages) == 4 assert Enum.all?(messages, fn msg -> msg.body == "ohai" end) end test "request_multi with no_responders" do topic = "nobody.is.home" {:ok, pid} = Gnat.start_link(%{no_responders: true}) assert {:error, :no_responders} = Gnat.request_multi(pid, topic, "ohai", max_messages: 2) end defp spin_up_echo_server_on_topic(ready, gnat, topic) do spawn(fn -> {:ok, subscription} = Gnat.sub(gnat, self(), topic) :ok = Gnat.unsub(gnat, subscription, max_messages: 1) send(ready, true) receive do {:msg, %{topic: ^topic, body: body, reply_to: reply_to, headers: headers}} -> Gnat.pub(gnat, reply_to, body, headers: headers) {:msg, %{topic: ^topic, body: body, reply_to: reply_to}} -> Gnat.pub(gnat, reply_to, body) end end) end test "recording errors from the broker" do import ExUnit.CaptureLog {:ok, gnat} = Gnat.start_link() assert capture_log(fn -> Process.flag(:trap_exit, true) Gnat.sub(gnat, self(), "invalid. subject") # errors are reported asynchronously so we need to wait a moment Process.sleep(20) end) =~ "Invalid Subject" end test "connection timeout" do start = System.monotonic_time(:millisecond) connection_settings = %{host: ~c"169.33.33.33", connection_timeout: 200} {:stop, :timeout} = Gnat.init(connection_settings) assert_in_delta System.monotonic_time(:millisecond) - start, 200, 10 end test "request-reply with custom inbox prefix" do topic = "req-resp" {:ok, pid} = Gnat.start_link(%{inbox_prefix: "custom._INBOX."}) spin_up_echo_server_on_topic(self(), pid, topic) # Wait for server to spawn and subscribe. assert_receive(true, 100) headers = [{"accept", "json"}] {:ok, msg} = Gnat.request(pid, topic, "ohai", receive_timeout: 500, headers: headers) assert "custom._INBOX." <> _ = msg.topic end test "server_info/1 returns server info" do {:ok, pid} = Gnat.start_link() info = Gnat.server_info(pid) assert Map.has_key?(info, :version) assert is_binary(info.version) end end ================================================ FILE: test/jetstream/api/consumer_doc_test.exs ================================================ defmodule Gnat.Jetstream.API.ConsumerDocTest do use Gnat.Jetstream.ConnCase @moduletag with_gnat: :gnat doctest Gnat.Jetstream.API.Consumer end ================================================ FILE: test/jetstream/api/consumer_test.exs ================================================ defmodule Gnat.Jetstream.API.ConsumerTest do use Gnat.Jetstream.ConnCase alias Gnat.Jetstream.API.{Consumer, Stream} @moduletag with_gnat: :gnat test "listing, creating, and deleting consumers" do stream = %Stream{name: "STREAM1", subjects: ["STREAM1"]} {:ok, _response} = Stream.create(:gnat, stream) assert {:ok, consumers} = Consumer.list(:gnat, "STREAM1") assert consumers == %{ total: 0, offset: 0, limit: 1024, consumers: [] } consumer = %Consumer{stream_name: "STREAM1", durable_name: "STREAM1"} assert {:ok, consumer_response} = Consumer.create(:gnat, consumer) assert consumer_response.ack_floor == %{ consumer_seq: 0, stream_seq: 0 } assert consumer_response.delivered == %{ consumer_seq: 0, stream_seq: 0 } assert %DateTime{} = consumer_response.created assert consumer_response.config == %{ ack_policy: :explicit, ack_wait: 30_000_000_000, deliver_policy: :all, deliver_subject: nil, durable_name: "STREAM1", filter_subject: nil, opt_start_seq: nil, opt_start_time: nil, replay_policy: :instant, backoff: nil, deliver_group: nil, description: nil, flow_control: nil, headers_only: nil, idle_heartbeat: nil, inactive_threshold: nil, max_ack_pending: 20000, max_batch: nil, max_deliver: -1, max_expires: nil, max_waiting: 512, rate_limit_bps: nil, sample_freq: nil } assert consumer_response.num_pending == 0 assert consumer_response.num_redelivered == 0 assert {:ok, consumers} = Consumer.list(:gnat, "STREAM1") assert consumers == %{ total: 1, offset: 0, limit: 1024, consumers: ["STREAM1"] } assert :ok = Consumer.delete(:gnat, "STREAM1", "STREAM1") assert {:ok, consumers} = Consumer.list(:gnat, "STREAM1") assert consumers == %{ total: 0, offset: 0, limit: 1024, consumers: [] } end test "failed creates" do consumer = %Consumer{durable_name: "STREAM2", stream_name: "STREAM2"} assert {:error, %{"code" => 404, "description" => "stream not found"}} = Consumer.create(:gnat, consumer) end test "failed deletes" do assert {:error, %{"code" => 404, "description" => "stream not found"}} = Consumer.delete(:gnat, "STREAM3", "STREAM3") end test "getting consumer info" do stream = %Stream{name: "STREAM4", subjects: ["STREAM4"]} {:ok, _response} = Stream.create(:gnat, stream) consumer = %Consumer{ stream_name: "STREAM4", durable_name: "STREAM4", deliver_subject: "consumer.STREAM4" } assert {:ok, _consumer_response} = Consumer.create(:gnat, consumer) assert {:ok, consumer_response} = Consumer.info(:gnat, "STREAM4", "STREAM4") assert consumer_response.ack_floor == %{ consumer_seq: 0, stream_seq: 0 } assert consumer_response.delivered == %{ consumer_seq: 0, stream_seq: 0 } assert %DateTime{} = consumer_response.created assert consumer_response.config == %{ ack_policy: :explicit, ack_wait: 30_000_000_000, deliver_policy: :all, deliver_subject: "consumer.STREAM4", durable_name: "STREAM4", filter_subject: nil, opt_start_seq: nil, opt_start_time: nil, replay_policy: :instant, backoff: nil, deliver_group: nil, description: nil, flow_control: nil, headers_only: nil, idle_heartbeat: nil, inactive_threshold: nil, max_ack_pending: 20000, max_batch: nil, max_deliver: -1, max_expires: nil, max_waiting: nil, rate_limit_bps: nil, sample_freq: nil } assert consumer_response.num_pending == 0 assert consumer_response.num_redelivered == 0 assert :ok = Consumer.delete(:gnat, "STREAM4", "STREAM4") assert :ok = Stream.delete(:gnat, "STREAM4") end test "validating stream and consumer names" do assert {:error, reason} = Consumer.create(:gnat, %Consumer{stream_name: "test.periods", durable_name: "foo"}) assert reason == "invalid stream_name: cannot contain '.', '>', '*', spaces or tabs" assert {:error, reason} = Consumer.create(:gnat, %Consumer{stream_name: nil, durable_name: "foo"}) assert reason == "must have a :stream_name set" assert {:error, reason} = Consumer.create(:gnat, %Consumer{stream_name: :foo, durable_name: "foo"}) assert reason == "stream_name must be a string" assert {:error, reason} = Consumer.create(:gnat, %Consumer{stream_name: "TEST_STREAM", durable_name: "foo.bar"}) assert reason == "invalid durable_name: cannot contain '.', '>', '*', spaces or tabs" assert {:error, reason} = Consumer.create(:gnat, %Consumer{stream_name: "TEST_STREAM", durable_name: :ohai}) assert reason == "durable_name must be a string" end describe "request_next_message/5" do setup do stream_name = "REQUEST_MESSAGE_TEST_STREAM" subject = "request_test_subject" consumer_name = "REQUEST_MESSAGE_TEST_CONSUMER" Stream.delete(:gnat, stream_name) stream = %Stream{name: stream_name, subjects: [subject]} {:ok, _response} = Stream.create(:gnat, stream) consumer = %Consumer{stream_name: stream_name, durable_name: consumer_name} assert {:ok, _response} = Consumer.create(:gnat, consumer) reply_subject = "reply" Gnat.sub(:gnat, self(), reply_subject) %{ stream_name: stream_name, subject: subject, consumer_name: consumer_name, reply_subject: reply_subject } end test "requests a single message with default options", %{ stream_name: stream_name, subject: subject, consumer_name: consumer_name, reply_subject: reply_subject } do Consumer.request_next_message(:gnat, stream_name, consumer_name, reply_subject) Gnat.pub(:gnat, subject, "message 1") assert_receive {:msg, %{body: "message 1", topic: ^subject}} Gnat.pub(:gnat, subject, "message 2") refute_receive {:msg, %{body: "message 2"}} end test "requests batch messages", %{ stream_name: stream_name, subject: subject, consumer_name: consumer_name, reply_subject: reply_subject } do Consumer.request_next_message(:gnat, stream_name, consumer_name, reply_subject, nil, batch: 10 ) Gnat.pub(:gnat, subject, "message 1") assert_receive {:msg, %{body: "message 1", topic: ^subject}} for i <- 2..10, do: Gnat.pub(:gnat, subject, "message #{i}") for i <- 2..10 do expected_body = "message #{i}" assert_receive {:msg, %{body: ^expected_body, topic: ^subject}} end Gnat.pub(:gnat, subject, "message 11") refute_receive {:msg, %{body: "message 11"}} end test "doesn't wait for messages when `no_wait` option is set to true", %{ stream_name: stream_name, subject: subject, consumer_name: consumer_name, reply_subject: reply_subject } do Consumer.request_next_message(:gnat, stream_name, consumer_name, reply_subject, nil, no_wait: true ) assert_receive {:msg, %{body: "", topic: ^reply_subject}} Gnat.pub(:gnat, subject, "message 1") refute_receive {:msg, %{body: "message 1"}} end test "doesn't wait for messages to complete the batch size with `no_wait`", %{ stream_name: stream_name, subject: subject, consumer_name: consumer_name, reply_subject: reply_subject } do for i <- 1..9 do # Using `request` to make sure messages get in the stream before we move on to consume them assert {:ok, _} = Gnat.request(:gnat, subject, "message #{i}") end Consumer.request_next_message(:gnat, stream_name, consumer_name, reply_subject, nil, batch: 10, no_wait: true ) for i <- 1..9 do expected_body = "message #{i}" assert_receive {:msg, %{body: ^expected_body, topic: ^subject}}, 2_000 end assert_receive {:msg, %{body: "", topic: ^reply_subject}}, 2_000 Gnat.pub(:gnat, subject, "message 10") refute_receive {:msg, %{body: "message 10"}} end end end ================================================ FILE: test/jetstream/api/kv/entry_test.exs ================================================ defmodule Gnat.Jetstream.API.KV.EntryTest do use ExUnit.Case, async: true alias Gnat.Jetstream.API.KV.Entry @bucket "my_bucket" @reply_to "$JS.ACK.KV_my_bucket.consumer.1.42.7.1750928948439739269.3" describe "from_message/2" do test "parses a put message with no headers" do message = %{topic: "$KV.my_bucket.some.key", body: "hello", reply_to: @reply_to} assert {:ok, entry} = Entry.from_message(message, @bucket) assert %Entry{ bucket: "my_bucket", key: "some.key", value: "hello", operation: :put, revision: 42, delta: 3 } = entry assert %DateTime{} = entry.created end test "parses a put message when headers are present but not KV-operation" do message = %{ topic: "$KV.my_bucket.foo", body: "bar", headers: [{"some-other", "value"}] } assert {:ok, %Entry{operation: :put, key: "foo", value: "bar"}} = Entry.from_message(message, @bucket) end test "parses a DEL message as :delete" do message = %{ topic: "$KV.my_bucket.foo", body: "", headers: [{"kv-operation", "DEL"}] } assert {:ok, %Entry{operation: :delete, key: "foo", value: ""}} = Entry.from_message(message, @bucket) end test "parses a PURGE message as :purge" do message = %{ topic: "$KV.my_bucket.foo", body: "", headers: [{"kv-operation", "PURGE"}] } assert {:ok, %Entry{operation: :purge, key: "foo"}} = Entry.from_message(message, @bucket) end test "treats a nats-marker-reason tombstone as :delete" do message = %{ topic: "$KV.my_bucket.foo", body: "", headers: [{"nats-marker-reason", "MaxAge"}] } assert {:ok, %Entry{operation: :delete, key: "foo"}} = Entry.from_message(message, @bucket) end test "recovers keys that include dots" do message = %{topic: "$KV.my_bucket.a.b.c", body: "v"} assert {:ok, %Entry{key: "a.b.c"}} = Entry.from_message(message, @bucket) end test "leaves revision/created/delta nil when no reply_to is present" do message = %{topic: "$KV.my_bucket.foo", body: "bar"} assert {:ok, %Entry{revision: nil, created: nil, delta: nil}} = Entry.from_message(message, @bucket) end test "returns :ignore when the subject does not belong to the bucket" do message = %{topic: "$KV.other_bucket.foo", body: "bar"} assert :ignore = Entry.from_message(message, @bucket) end test "returns :ignore for a 409 leadership-change status message" do message = %{ topic: "_INBOX.foo", body: "", status: "409", description: "Leadership Change" } assert :ignore = Entry.from_message(message, @bucket) end test "returns :ignore for a 100 idle heartbeat with no description" do message = %{topic: "_INBOX.foo", body: "", status: "100"} assert :ignore = Entry.from_message(message, @bucket) end end end ================================================ FILE: test/jetstream/api/kv/watcher_test.exs ================================================ defmodule Gnat.Jetstream.API.KV.WatcherTest do use Gnat.Jetstream.ConnCase, min_server_version: "2.6.2" alias Gnat.Jetstream.API.KV @moduletag with_gnat: :gnat describe "status messages" do setup do bucket = "WATCHER_STATUS_TEST" {:ok, _} = KV.create_bucket(:gnat, bucket) on_exit(fn -> {:ok, pid} = Gnat.start_link() :ok = KV.delete_bucket(pid, bucket) Gnat.stop(pid) end) test_pid = self() {:ok, watcher} = KV.watch(:gnat, bucket, fn action, key, value -> send(test_pid, {action, key, value}) end) on_exit(fn -> if Process.alive?(watcher), do: KV.unwatch(watcher) end) %{watcher: watcher} end test "409 status messages are dropped", %{watcher: watcher} do send( watcher, {:msg, %{status: "409", description: "Leadership Change", body: "", topic: "ignored"}} ) refute_receive {:key_added, _, _}, 100 refute_receive {:key_deleted, _, _}, 100 assert Process.alive?(watcher) end test "100 idle heartbeat (no reply_to) is dropped", %{watcher: watcher} do send(watcher, {:msg, %{status: "100", body: "", topic: "ignored"}}) refute_receive {:key_added, _, _}, 100 assert Process.alive?(watcher) end test "100 flow-control message is answered with an empty publish", %{watcher: watcher} do reply_to = "_WATCHER_TEST.fc.#{System.unique_integer([:positive])}" {:ok, _sub} = Gnat.sub(:gnat, self(), reply_to) send( watcher, {:msg, %{ status: "100", description: "FlowControl Request", body: "", reply_to: reply_to, topic: "ignored" }} ) assert_receive {:msg, %{topic: ^reply_to, body: ""}}, 500 end end end ================================================ FILE: test/jetstream/api/kv_test.exs ================================================ defmodule Gnat.Jetstream.API.KVTest do use Gnat.Jetstream.ConnCase, min_server_version: "2.6.2" alias Gnat.Jetstream.API.KV alias Gnat.Jetstream.API.Stream @moduletag with_gnat: :gnat describe "create_bucket/3" do test "creates a bucket" do assert {:ok, %{config: config}} = KV.create_bucket(:gnat, "BUCKET_TEST") assert config.name == "KV_BUCKET_TEST" assert config.subjects == ["$KV.BUCKET_TEST.>"] assert config.max_msgs_per_subject == 1 assert config.discard == :new assert config.allow_rollup_hdrs == true assert :ok = KV.delete_bucket(:gnat, "BUCKET_TEST") end test "creates a bucket with duplicate window < 2min" do assert {:ok, %{config: config}} = KV.create_bucket(:gnat, "TTL_TEST", ttl: 1_000_000_000) assert config.max_age == 1_000_000_000 assert config.duplicate_window == 1_000_000_000 assert :ok = KV.delete_bucket(:gnat, "TTL_TEST") end test "creates a bucket with duplicate window > 2min" do assert {:ok, %{config: config}} = KV.create_bucket(:gnat, "OTHER_TTL_TEST", ttl: 130_000_000_000) assert config.max_age == 130_000_000_000 assert config.duplicate_window == 120_000_000_000 assert :ok = KV.delete_bucket(:gnat, "OTHER_TTL_TEST") end @tag :message_ttl test "creates a bucket with limit_marker_ttl" do assert {:ok, %{config: config}} = KV.create_bucket(:gnat, "LIMIT_MARKER_TTL_TEST", limit_marker_ttl: 1_000_000_000) assert config.subject_delete_marker_ttl == 1_000_000_000 assert :ok = KV.delete_bucket(:gnat, "LIMIT_MARKER_TTL_TEST") end end test "create_key/4 creates a key" do assert {:ok, _} = KV.create_bucket(:gnat, "KEY_CREATE_TEST") assert :ok = KV.create_key(:gnat, "KEY_CREATE_TEST", "foo", "bar") assert "bar" = KV.get_value(:gnat, "KEY_CREATE_TEST", "foo") assert :ok = KV.delete_bucket(:gnat, "KEY_CREATE_TEST") end test "create_key/4 returns error" do assert {:error, :timeout} = KV.create_key(:gnat, "KEY_CREATE_TEST", "foo", "bar", timeout: 1) end test "delete_key/3 deletes a key" do assert {:ok, _} = KV.create_bucket(:gnat, "KEY_DELETE_TEST") assert :ok = KV.create_key(:gnat, "KEY_DELETE_TEST", "foo", "bar") assert :ok = KV.delete_key(:gnat, "KEY_DELETE_TEST", "foo") assert KV.get_value(:gnat, "KEY_DELETE_TEST", "foo") == nil assert :ok = KV.delete_bucket(:gnat, "KEY_DELETE_TEST") end test "delete_key/3 returns error" do assert {:error, :timeout} = KV.delete_key(:gnat, "KEY_DELETE_TEST", "foo", timeout: 1) end test "purge_key/3 purges a key" do assert {:ok, _} = KV.create_bucket(:gnat, "KEY_PURGE_TEST") assert :ok = KV.create_key(:gnat, "KEY_PURGE_TEST", "foo", "bar") assert :ok = KV.purge_key(:gnat, "KEY_PURGE_TEST", "foo") assert KV.get_value(:gnat, "KEY_PURGE_TEST", "foo") == nil assert :ok = KV.delete_bucket(:gnat, "KEY_PURGE_TEST") end test "purge_key/3 returns error" do assert {:error, :timeout} = KV.purge_key(:gnat, "KEY_PURGE_TEST", "foo", timeout: 1) end test "put_value/4 updates a key" do assert {:ok, _} = KV.create_bucket(:gnat, "KEY_PUT_TEST") assert :ok = KV.create_key(:gnat, "KEY_PUT_TEST", "foo", "bar") assert :ok = KV.put_value(:gnat, "KEY_PUT_TEST", "foo", "baz") assert "baz" = KV.get_value(:gnat, "KEY_PUT_TEST", "foo") assert :ok = KV.delete_bucket(:gnat, "KEY_PUT_TEST") end test "put_value/4 returns error" do assert {:error, :timeout} = KV.put_value(:gnat, "KEY_PUT_TEST", "foo", "baz", timeout: 1) end @tag :message_ttl test "detects key removed based on limit_marker_ttl" do assert {:ok, _} = KV.create_bucket(:gnat, "LIMIT_MARKER_TTL_TEST", limit_marker_ttl: 1_000_000_000, ttl: 1_000_000_000 ) test_pid = self() {:ok, watcher_pid} = KV.watch(:gnat, "LIMIT_MARKER_TTL_TEST", fn action, key, value -> send(test_pid, {action, key, value}) end) KV.put_value(:gnat, "LIMIT_MARKER_TTL_TEST", "foo", "bar") assert_receive({:key_added, "foo", "bar"}) assert_receive({:key_deleted, "foo", ""}, 1500) KV.unwatch(watcher_pid) assert :ok = KV.delete_bucket(:gnat, "LIMIT_MARKER_TTL_TEST") end describe "watch/3" do setup do bucket = "KEY_WATCH_TEST" {:ok, _} = KV.create_bucket(:gnat, bucket) %{bucket: bucket} end test "detects key added and removed keys", %{bucket: bucket} do test_pid = self() {:ok, watcher_pid} = KV.watch(:gnat, bucket, fn action, key, value -> send(test_pid, {action, key, value}) end) KV.put_value(:gnat, bucket, "foo", "bar") assert_receive({:key_added, "foo", "bar"}) KV.put_value(:gnat, bucket, "baz", "quz") assert_receive({:key_added, "baz", "quz"}) KV.delete_key(:gnat, bucket, "baz") # key deletions don't carry the data removed assert_receive({:key_deleted, "baz", ""}) KV.put_value(:gnat, bucket, "foo", "buzz") assert_receive({:key_added, "foo", "buzz"}) KV.purge_key(:gnat, bucket, "foo") assert_receive({:key_purged, "foo", ""}) KV.unwatch(watcher_pid) :ok = KV.delete_bucket(:gnat, bucket) end end describe "contents/2" do setup do bucket = "KEY_LIST_TEST" {:ok, _} = KV.create_bucket(:gnat, bucket) %{bucket: bucket} end test "provides all keys", %{bucket: bucket} do KV.put_value(:gnat, bucket, "foo", "bar") KV.put_value(:gnat, bucket, "baz", "quz") assert {:ok, %{"foo" => "bar", "baz" => "quz"}} == KV.contents(:gnat, bucket) :ok = KV.delete_bucket(:gnat, bucket) end test "deleted keys not included", %{bucket: bucket} do KV.put_value(:gnat, bucket, "foo", "bar") KV.put_value(:gnat, bucket, "baz", "quz") KV.delete_key(:gnat, bucket, "baz") assert {:ok, %{"foo" => "bar"}} == KV.contents(:gnat, bucket) :ok = KV.delete_bucket(:gnat, bucket) end test "updated keys use most recent", %{bucket: bucket} do :ok = KV.delete_bucket(:gnat, bucket) {:ok, _} = KV.create_bucket(:gnat, bucket, history: 5) KV.put_value(:gnat, bucket, "foo", "bar") KV.put_value(:gnat, bucket, "foo", "baz") assert {:ok, %{"foo" => "baz"}} == KV.contents(:gnat, bucket) :ok = KV.delete_bucket(:gnat, bucket) end test "empty for no keys", %{bucket: bucket} do assert {:ok, %{}} == KV.contents(:gnat, bucket) :ok = KV.delete_bucket(:gnat, bucket) end test "error tuple if problem", %{bucket: bucket} do assert {:error, _message} = KV.contents(:gnat, "NOT_REAL_BUCKET") :ok = KV.delete_bucket(:gnat, bucket) end end describe "keys/2" do setup do bucket = "KEY_KEYS_TEST" {:ok, _} = KV.create_bucket(:gnat, bucket) %{bucket: bucket} end test "provides all keys", %{bucket: bucket} do KV.put_value(:gnat, bucket, "foo", "bar") KV.put_value(:gnat, bucket, "baz", "quz") KV.put_value(:gnat, bucket, "alpha", "beta") assert {:ok, ["alpha", "baz", "foo"]} == KV.keys(:gnat, bucket) :ok = KV.delete_bucket(:gnat, bucket) end test "deleted keys not included", %{bucket: bucket} do KV.put_value(:gnat, bucket, "foo", "bar") KV.put_value(:gnat, bucket, "baz", "quz") KV.put_value(:gnat, bucket, "alpha", "beta") KV.delete_key(:gnat, bucket, "baz") assert {:ok, ["alpha", "foo"]} == KV.keys(:gnat, bucket) :ok = KV.delete_bucket(:gnat, bucket) end test "purged keys not included", %{bucket: bucket} do KV.put_value(:gnat, bucket, "foo", "bar") KV.put_value(:gnat, bucket, "baz", "quz") KV.purge_key(:gnat, bucket, "foo") assert {:ok, ["baz"]} == KV.keys(:gnat, bucket) :ok = KV.delete_bucket(:gnat, bucket) end test "updated keys only appear once", %{bucket: bucket} do :ok = KV.delete_bucket(:gnat, bucket) {:ok, _} = KV.create_bucket(:gnat, bucket, history: 5) KV.put_value(:gnat, bucket, "foo", "bar") KV.put_value(:gnat, bucket, "foo", "baz") KV.put_value(:gnat, bucket, "foo", "qux") assert {:ok, ["foo"]} == KV.keys(:gnat, bucket) :ok = KV.delete_bucket(:gnat, bucket) end test "empty list for no keys", %{bucket: bucket} do assert {:ok, []} == KV.keys(:gnat, bucket) :ok = KV.delete_bucket(:gnat, bucket) end test "keys are sorted alphabetically", %{bucket: bucket} do KV.put_value(:gnat, bucket, "zebra", "value1") KV.put_value(:gnat, bucket, "apple", "value2") KV.put_value(:gnat, bucket, "middle", "value3") assert {:ok, ["apple", "middle", "zebra"]} == KV.keys(:gnat, bucket) :ok = KV.delete_bucket(:gnat, bucket) end test "error tuple if bucket does not exist", %{bucket: bucket} do assert {:error, _message} = KV.keys(:gnat, "NOT_REAL_BUCKET") :ok = KV.delete_bucket(:gnat, bucket) end test "handles keys with special characters", %{bucket: bucket} do KV.put_value(:gnat, bucket, "key.with.dots", "value1") KV.put_value(:gnat, bucket, "key-with-dashes", "value2") KV.put_value(:gnat, bucket, "key_with_underscores", "value3") assert {:ok, ["key-with-dashes", "key.with.dots", "key_with_underscores"]} == KV.keys(:gnat, bucket) :ok = KV.delete_bucket(:gnat, bucket) end end describe "list_buckets/2" do test "list buckets when none exists" do assert {:ok, []} = KV.list_buckets(:gnat) end test "list buckets properly" do assert {:ok, %{config: _config}} = KV.create_bucket(:gnat, "TEST_BUCKET_1") assert {:ok, %{config: _config}} = KV.create_bucket(:gnat, "TEST_BUCKET_2") assert {:ok, ["TEST_BUCKET_1", "TEST_BUCKET_2"]} = KV.list_buckets(:gnat) :ok = KV.delete_bucket(:gnat, "TEST_BUCKET_1") :ok = KV.delete_bucket(:gnat, "TEST_BUCKET_2") end test "ignore streams that are not buckets" do assert {:ok, %{config: _config}} = KV.create_bucket(:gnat, "TEST_BUCKET_1") stream = %Stream{ name: "TEST_STREAM_1", subjects: ["TEST_STREAM_1.subject1", "TEST_STREAM_1.subject2"] } assert {:ok, _response} = Stream.create(:gnat, stream) assert {:ok, ["TEST_BUCKET_1"]} = KV.list_buckets(:gnat) :ok = KV.delete_bucket(:gnat, "TEST_BUCKET_1") end end describe "info/3" do test "returns bucket info" do assert {:ok, _} = KV.create_bucket(:gnat, "TEST_BUCKET_1") assert {:ok, %{config: %{name: "KV_TEST_BUCKET_1"}}} = KV.info(:gnat, "TEST_BUCKET_1") :ok = KV.delete_bucket(:gnat, "TEST_BUCKET_1") assert {:error, %{"code" => 404}} = KV.info(:gnat, "NOT_A_BUCKET") end end end ================================================ FILE: test/jetstream/api/object_test.exs ================================================ defmodule Gnat.Jetstream.API.ObjectTest do use Gnat.Jetstream.ConnCase, min_server_version: "2.6.2" alias Gnat.Jetstream.API.{Object, Stream} import Gnat.Jetstream.API.Util, only: [nuid: 0] @moduletag with_gnat: :gnat @changelog_path Path.join([Path.dirname(__DIR__), "..", "..", "CHANGELOG.md"]) @readme_path Path.join([Path.dirname(__DIR__), "..", "..", "README.md"]) describe "create_bucket/3" do test "create/delete a bucket" do assert {:ok, %{config: config}} = Object.create_bucket(:gnat, "MY-STORE") assert config.name == "OBJ_MY-STORE" assert config.max_age == 0 assert config.max_bytes == -1 assert config.storage == :file assert config.allow_rollup_hdrs == true assert config.subjects == [ "$O.MY-STORE.C.>", "$O.MY-STORE.M.>" ] assert :ok = Object.delete_bucket(:gnat, "MY-STORE") end test "creating a bucket with TTL" do bucket = nuid() # 10s in nanoseconds ttl = 10 * 1_000_000_000 assert {:ok, %{config: config}} = Object.create_bucket(:gnat, bucket, ttl: ttl) assert config.max_age == ttl assert :ok = Object.delete_bucket(:gnat, bucket) end test "bucket names are validated" do assert {:error, "invalid bucket name"} = Object.create_bucket(:gnat, "") assert {:error, "invalid bucket name"} = Object.create_bucket(:gnat, "MY.STORE") assert {:error, "invalid bucket name"} = Object.create_bucket(:gnat, "(*!&@($%*&))") end end describe "delete_bucket/2" do test "create/delete a bucket" do assert {:ok, %{config: _config}} = Object.create_bucket(:gnat, "MY-STORE") assert :ok = Object.delete_bucket(:gnat, "MY-STORE") end end describe "delete/3" do test "delete an object" do bucket = nuid() assert {:ok, %{config: _config}} = Object.create_bucket(:gnat, bucket) {:ok, _} = put_filepath(@readme_path, bucket, "README.md") {:ok, _} = put_filepath(@readme_path, bucket, "OTHER.md") assert :ok = Object.delete(:gnat, bucket, "README.md") assert {:ok, objects} = Object.list(:gnat, bucket) assert Enum.count(objects) == 1 assert Enum.map(objects, & &1.name) == ["OTHER.md"] assert {:ok, objects} = Object.list(:gnat, bucket, show_deleted: true) assert Enum.count(objects) == 2 assert Enum.map(objects, & &1.name) |> Enum.sort() == ["OTHER.md", "README.md"] assert :ok = Object.delete_bucket(:gnat, bucket) end end describe "get/4" do test "retrieves and object chunk-by-chunk" do nuid = nuid() assert {:ok, _} = Object.create_bucket(:gnat, nuid) readme_content = File.read!(@readme_path) assert {:ok, _meta} = put_filepath(@readme_path, nuid, "README.md") assert :ok = Object.get(:gnat, nuid, "README.md", fn chunk -> assert chunk == readme_content send(self(), :got_chunk) end) assert_received :got_chunk :ok = Object.delete_bucket(:gnat, nuid) end end describe "info/3" do test "lookup meta information about an object" do assert {:ok, %{config: _stream}} = Object.create_bucket(:gnat, "INF") assert {:ok, io} = File.open(@readme_path, [:read]) assert {:ok, initial_meta} = Object.put(:gnat, "INF", "README.md", io) assert {:ok, lookup_meta} = Object.info(:gnat, "INF", "README.md") assert lookup_meta == initial_meta assert :ok = Object.delete_bucket(:gnat, "INF") end end describe "list/3" do test "list an empty bucket" do bucket = nuid() assert {:ok, %{config: _config}} = Object.create_bucket(:gnat, bucket) assert {:ok, []} = Object.list(:gnat, bucket) assert :ok = Object.delete_bucket(:gnat, bucket) end test "list a bucket with two files" do bucket = nuid() assert {:ok, %{config: _config}} = Object.create_bucket(:gnat, bucket) assert {:ok, io} = File.open(@readme_path, [:read]) assert {:ok, _object} = Object.put(:gnat, bucket, "README.md", io) assert {:ok, io} = File.open(@readme_path, [:read]) assert {:ok, _object} = Object.put(:gnat, bucket, "SOMETHING.md", io) assert {:ok, objects} = Object.list(:gnat, bucket) [readme, something] = Enum.sort_by(objects, & &1.name) assert readme.name == "README.md" assert readme.size == something.size assert readme.digest == something.digest assert :ok = Object.delete_bucket(:gnat, bucket) end end describe "put/4" do test "creates an object" do assert {:ok, %{config: _stream}} = Object.create_bucket(:gnat, "MY-STORE") expected_sha = @readme_path |> File.read!() |> then(&:crypto.hash(:sha256, &1)) assert {:ok, object_meta} = put_filepath(@readme_path, "MY-STORE", "README.md") assert object_meta.name == "README.md" assert object_meta.bucket == "MY-STORE" assert object_meta.chunks == 1 assert "SHA-256=" <> encoded = object_meta.digest assert Base.url_decode64!(encoded) == expected_sha assert :ok = Object.delete_bucket(:gnat, "MY-STORE") end test "overwriting a file" do bucket = nuid() assert {:ok, %{config: _stream}} = Object.create_bucket(:gnat, bucket) assert {:ok, _} = put_filepath(@readme_path, bucket, "WAT") size_after_readme = stream_byte_size(bucket) assert {:ok, _} = put_filepath(@changelog_path, bucket, "WAT") size_after_changelog = stream_byte_size(bucket) assert size_after_changelog < size_after_readme assert {:ok, [meta]} = Object.list(:gnat, bucket) assert meta.name == "WAT" assert :ok = Object.delete_bucket(:gnat, bucket) end test "return an error if the object store doesn't exist" do assert {:error, err} = put_filepath(@readme_path, "I_DONT_EXIST", "foo") assert %{"code" => 404, "description" => "stream not found"} = err end end @tag :tmp_dir test "storing and retrieving larger files", %{tmp_dir: tmp_dir} do assert {:ok, path, sha} = generate_big_file(tmp_dir) bucket = nuid() assert {:ok, %{config: _stream}} = Object.create_bucket(:gnat, bucket) assert {:ok, meta} = put_filepath(path, bucket, "big") assert meta.chunks == 800 assert meta.size == 800 * 128 * 1024 assert "SHA-256=" <> encoded = meta.digest assert Base.url_decode64!(encoded) == sha Process.put(:buffer, "") Object.get(:gnat, bucket, "big", fn chunk -> Process.put(:buffer, Process.get(:buffer) <> chunk) end) file_contents = Process.get(:buffer) assert byte_size(file_contents) == meta.size assert :crypto.hash(:sha256, file_contents) == sha assert stream_byte_size(bucket) > 1024 * 1024 assert :ok = Object.delete(:gnat, bucket, "big") assert stream_byte_size(bucket) < 1024 :ok = Object.delete_bucket(:gnat, bucket) end @tag :tmp_dir test "control messages don't affect chunk count", %{tmp_dir: tmp_dir} do assert {:ok, path, _sha} = generate_big_file(tmp_dir) bucket = nuid() assert {:ok, %{config: _stream}} = Object.create_bucket(:gnat, bucket) assert {:ok, meta} = put_filepath(path, bucket, "test_chunks") Process.put(:chunk_count, 0) :ok = Object.get(:gnat, bucket, "test_chunks", fn _chunk -> Process.put(:chunk_count, Process.get(:chunk_count) + 1) end) chunk_count = Process.get(:chunk_count) assert chunk_count == meta.chunks :ok = Object.delete_bucket(:gnat, bucket) end describe "list_buckets/2" do test "list buckets when none exists" do assert {:ok, []} = Object.list_buckets(:gnat) end test "list buckets properly" do assert {:ok, %{config: _config}} = Object.create_bucket(:gnat, "TEST_BUCKET_1") assert {:ok, %{config: _config}} = Object.create_bucket(:gnat, "TEST_BUCKET_2") assert {:ok, ["TEST_BUCKET_1", "TEST_BUCKET_2"]} = Object.list_buckets(:gnat) :ok = Object.delete_bucket(:gnat, "TEST_BUCKET_1") :ok = Object.delete_bucket(:gnat, "TEST_BUCKET_2") end test "ignore streams that are not buckets" do assert {:ok, %{config: _config}} = Object.create_bucket(:gnat, "TEST_BUCKET_1") stream = %Stream{ name: "TEST_STREAM_1", subjects: ["TEST_STREAM_1.subject1", "TEST_STREAM_1.subject2"] } assert {:ok, _response} = Stream.create(:gnat, stream) assert {:ok, ["TEST_BUCKET_1"]} = Object.list_buckets(:gnat) :ok = Object.delete_bucket(:gnat, "TEST_BUCKET_1") end end # create a random 100MB binary file # re-use it on subsequent test runs if it already exists defp generate_big_file(tmp_dir) do filepath = Path.join(tmp_dir, "big_file.bin") sha = :crypto.hash_init(:sha256) {:ok, fh} = File.open(filepath, [:write]) sha = Enum.reduce(1..800, sha, fn _, digest -> rand_chunk = :crypto.strong_rand_bytes(128) |> String.duplicate(1024) :ok = IO.binwrite(fh, rand_chunk) :crypto.hash_update(digest, rand_chunk) end) :ok = File.close(fh) {:ok, filepath, :crypto.hash_final(sha)} end defp put_filepath(path, bucket, name) do {:ok, io} = File.open(path, [:read]) Object.put(:gnat, bucket, name, io) end defp stream_byte_size(bucket) do {:ok, %{state: state}} = Stream.info(:gnat, "OBJ_#{bucket}") state.bytes end end ================================================ FILE: test/jetstream/api/stream_doc_test.exs ================================================ defmodule Gnat.Jetstream.API.StreamDocTest do use Gnat.Jetstream.ConnCase @moduletag with_gnat: :gnat doctest Gnat.Jetstream.API.Stream end ================================================ FILE: test/jetstream/api/stream_test.exs ================================================ defmodule Gnat.Jetstream.API.StreamTest do use Gnat.Jetstream.ConnCase alias Gnat.Jetstream.API.Stream @moduletag with_gnat: :gnat test "listing and creating, and deleting streams" do {:ok, %{streams: streams}} = Stream.list(:gnat) assert streams == nil || !("LIST_TEST" in streams) stream = %Stream{name: "LIST_TEST", subjects: ["STREAM_TEST"]} {:ok, response} = Stream.create(:gnat, stream) assert response.config == stream assert response.state == %{ bytes: 0, consumer_count: 0, first_seq: 0, first_ts: ~U[0001-01-01 00:00:00Z], last_seq: 0, last_ts: ~U[0001-01-01 00:00:00Z], messages: 0, deleted: nil, lost: nil, num_deleted: nil, num_subjects: nil, subjects: nil } {:ok, %{streams: streams}} = Stream.list(:gnat) assert "LIST_TEST" in streams assert :ok = Stream.delete(:gnat, "LIST_TEST") {:ok, %{streams: streams}} = Stream.list(:gnat) assert streams == nil || !("LIST_TEST" in streams) end test "list/2 includes multiple streams" do stream = %Stream{name: "LIST_SUBJECT_TEST_ONE", subjects: ["LIST_SUBJECT_TEST.subject1"]} {:ok, _response} = Stream.create(:gnat, stream) stream = %Stream{name: "LIST_SUBJECT_TEST_TWO", subjects: ["LIST_SUBJECT_TEST.subject2"]} {:ok, _response} = Stream.create(:gnat, stream) {:ok, %{streams: streams}} = Stream.list(:gnat) assert "LIST_SUBJECT_TEST_ONE" in streams assert "LIST_SUBJECT_TEST_TWO" in streams assert :ok = Stream.delete(:gnat, "LIST_SUBJECT_TEST_ONE") assert :ok = Stream.delete(:gnat, "LIST_SUBJECT_TEST_TWO") end test "list/2 can filter by subject" do stream = %Stream{name: "LIST_SUBJECT_TEST_ONE", subjects: ["LIST_SUBJECT_TEST.subject1"]} {:ok, _response} = Stream.create(:gnat, stream) stream = %Stream{name: "LIST_SUBJECT_TEST_TWO", subjects: ["LIST_SUBJECT_TEST.subject2"]} {:ok, _response} = Stream.create(:gnat, stream) {:ok, %{streams: [stream]}} = Stream.list(:gnat, subject: "LIST_SUBJECT_TEST.subject2") assert stream == "LIST_SUBJECT_TEST_TWO" assert :ok = Stream.delete(:gnat, "LIST_SUBJECT_TEST_ONE") assert :ok = Stream.delete(:gnat, "LIST_SUBJECT_TEST_TWO") end test "list/2 includes accepts offset" do stream = %Stream{name: "LIST_OFFSET_TEST_ONE", subjects: ["LIST_OFFSET_TEST.subject1"]} {:ok, _response} = Stream.create(:gnat, stream) stream = %Stream{name: "LIST_OFFSET_TEST_TWO", subjects: ["LIST_OFFSET_TEST.subject2"]} {:ok, _response} = Stream.create(:gnat, stream) {:ok, %{streams: streams}} = Stream.list(:gnat) num_no_offset = Enum.count(streams) {:ok, %{streams: streams}} = Stream.list(:gnat, offset: 1) num_with_offset = Enum.count(streams) # Checking offset functionality without a strict pattern match to keep this # test passing even if another test forgets to delete a stream after it's done assert num_no_offset - num_with_offset == 1 assert :ok = Stream.delete(:gnat, "LIST_OFFSET_TEST_ONE") assert :ok = Stream.delete(:gnat, "LIST_OFFSET_TEST_TWO") end test "create stream with discard_new_per_subject: true" do stream = %Stream{ name: "DISCARD_NEW_PER_SUBJECT_TEST", subjects: ["STREAM_TEST"], max_msgs_per_subject: 1, discard_new_per_subject: true, discard: :new } assert {:ok, _response} = Stream.create(:gnat, stream) assert {:ok, _} = Gnat.request(:gnat, "STREAM_TEST", "first message") assert {:ok, response} = Stream.get_message(:gnat, "DISCARD_NEW_PER_SUBJECT_TEST", %{ last_by_subj: "STREAM_TEST" }) %{ data: "first message", hdrs: nil, subject: "STREAM_TEST", time: %DateTime{} } = response assert {:ok, _} = Gnat.request(:gnat, "STREAM_TEST", "second message") assert {:ok, response} = Stream.get_message(:gnat, "DISCARD_NEW_PER_SUBJECT_TEST", %{ last_by_subj: "STREAM_TEST" }) %{ data: "first message", hdrs: nil, subject: "STREAM_TEST", time: %DateTime{} } = response Stream.purge(:gnat, "DISCARD_NEW_PER_SUBJECT_TEST") assert {:ok, _} = Gnat.request(:gnat, "STREAM_TEST", "second message") assert {:ok, response} = Stream.get_message(:gnat, "DISCARD_NEW_PER_SUBJECT_TEST", %{ last_by_subj: "STREAM_TEST" }) %{ data: "second message", hdrs: nil, subject: "STREAM_TEST", time: %DateTime{} } = response assert :ok = Stream.delete(:gnat, "DISCARD_NEW_PER_SUBJECT_TEST") end @tag :message_ttl test "create a stream with messages TTL" do stream = %Stream{ name: "STREAM-ALLOW_MSG-TTL", subjects: ["STREAM_TTL_TEST"], allow_msg_ttl: true } assert {:ok, _response} = Stream.create(:gnat, stream) :ok = Gnat.pub(:gnat, "STREAM_TTL_TEST", "message-should-be-dropped-by-ttl", headers: [{"Nats-TTL", "1"}] ) {:ok, %{data: "message-should-be-dropped-by-ttl"}} = Stream.get_message(:gnat, "STREAM-ALLOW_MSG-TTL", %{last_by_subj: "STREAM_TTL_TEST"}) :timer.sleep(1500) {:error, %{"code" => 404}} = Stream.get_message(:gnat, "STREAM-ALLOW_MSG-TTL", %{last_by_subj: "STREAM_TTL_TEST"}) assert :ok = Stream.delete(:gnat, "STREAM-ALLOW_MSG-TTL") end @tag :message_ttl test "create a stream with subject delete marker TTL" do stream = %Stream{ name: "STREAM-SUBJECT-DELETE-MARKER-TTL", subjects: ["STREAM_TTL_TEST"], max_age: 1_000_000_000, duplicate_window: 1_000_000_000, subject_delete_marker_ttl: 1_000_000_000 } assert {:ok, _response} = Stream.create(:gnat, stream) :ok = Gnat.pub(:gnat, "STREAM_TTL_TEST", "message-should-be-dropped-by-ttl") {:ok, %{data: "message-should-be-dropped-by-ttl"}} = Stream.get_message(:gnat, "STREAM-SUBJECT-DELETE-MARKER-TTL", %{ last_by_subj: "STREAM_TTL_TEST" }) :timer.sleep(1500) {:ok, %{data: nil, hdrs: headers}} = Stream.get_message(:gnat, "STREAM-SUBJECT-DELETE-MARKER-TTL", %{ last_by_subj: "STREAM_TTL_TEST" }) assert true === String.contains?(headers, "Nats-Marker-Reason: MaxAge") assert :ok = Stream.delete(:gnat, "STREAM-SUBJECT-DELETE-MARKER-TTL") end test "updating a stream" do stream = %Stream{name: "UPDATE_TEST", subjects: ["STREAM_TEST"]} assert {:ok, _response} = Stream.create(:gnat, stream) updated_stream = %Stream{name: "UPDATE_TEST", subjects: ["STREAM_TEST", "NEW_SUBJECT"]} assert {:ok, response} = Stream.update(:gnat, updated_stream) assert response.config.subjects == ["STREAM_TEST", "NEW_SUBJECT"] assert :ok = Stream.delete(:gnat, "UPDATE_TEST") end test "failed deletes" do assert {:error, %{"code" => 404, "description" => "stream not found"}} = Stream.delete(:gnat, "NaN") end test "getting stream info" do stream = %Stream{name: "INFO_TEST", subjects: ["INFO_TEST.*"]} assert {:ok, _response} = Stream.create(:gnat, stream) assert {:ok, response} = Stream.info(:gnat, "INFO_TEST") assert response.config == stream assert response.state == %{ bytes: 0, consumer_count: 0, first_seq: 0, first_ts: ~U[0001-01-01 00:00:00Z], last_seq: 0, last_ts: ~U[0001-01-01 00:00:00Z], messages: 0, deleted: nil, lost: nil, num_deleted: nil, num_subjects: nil, subjects: nil } assert :ok = Stream.delete(:gnat, "INFO_TEST") end test "creating a stream with non-standard settings" do stream = %Stream{ name: "ARGS_TEST", subjects: ["ARGS_TEST.*"], retention: :workqueue, duplicate_window: 100_000_000, storage: :memory, compression: "s2" } assert {:ok, %{config: result}} = Stream.create(:gnat, stream) assert result.name == "ARGS_TEST" assert result.duplicate_window == 100_000_000 assert result.retention == :workqueue assert result.storage == :memory assert result.compression == "s2" assert :ok = Stream.delete(:gnat, "ARGS_TEST") end test "validating stream names" do assert {:error, reason} = Stream.create(:gnat, %Stream{name: "test.periods", subjects: ["foo"]}) assert reason == "invalid name: cannot contain '.', '>', '*', spaces or tabs" assert {:error, reason} = Stream.create(:gnat, %Stream{name: "test>greater", subjects: ["foo"]}) assert reason == "invalid name: cannot contain '.', '>', '*', spaces or tabs" assert {:error, reason} = Stream.create(:gnat, %Stream{name: "test_star*", subjects: ["foo"]}) assert reason == "invalid name: cannot contain '.', '>', '*', spaces or tabs" assert {:error, reason} = Stream.create(:gnat, %Stream{name: "test-space ", subjects: ["foo"]}) assert reason == "invalid name: cannot contain '.', '>', '*', spaces or tabs" assert {:error, reason} = Stream.create(:gnat, %Stream{name: "\ttest-tab", subjects: ["foo"]}) assert reason == "invalid name: cannot contain '.', '>', '*', spaces or tabs" end describe "get_message/3" do test "error if both seq and last_by_subj are used" do assert {:error, reason} = Stream.get_message(:gnat, "foo", %{seq: 1, last_by_subj: "bar"}) assert reason == "To get a message you must use only one of `seq` or `last_by_subj`" end test "decodes message data" do stream = %Stream{name: "GET_MESSAGE_TEST", subjects: ["GET_MESSAGE_TEST.foo"]} assert {:ok, _response} = Stream.create(:gnat, stream) assert {:ok, _} = Gnat.request(:gnat, "GET_MESSAGE_TEST.foo", "hi there") assert {:ok, response} = Stream.get_message(:gnat, "GET_MESSAGE_TEST", %{ last_by_subj: "GET_MESSAGE_TEST.foo" }) %{ data: "hi there", hdrs: nil, subject: "GET_MESSAGE_TEST.foo", time: %DateTime{} } = response assert is_number(response.seq) assert :ok = Stream.delete(:gnat, "GET_MESSAGE_TEST") end test "decodes message data with headers" do stream = %Stream{ name: "GET_MESSAGE_TEST_WITH_HEADERS", subjects: ["GET_MESSAGE_TEST_WITH_HEADERS.bar"] } assert {:ok, _response} = Stream.create(:gnat, stream) assert {:ok, %{body: _body}} = Gnat.request(:gnat, "GET_MESSAGE_TEST_WITH_HEADERS.bar", "hi there", headers: [{"foo", "bar"}] ) assert {:ok, response} = Stream.get_message(:gnat, "GET_MESSAGE_TEST_WITH_HEADERS", %{ last_by_subj: "GET_MESSAGE_TEST_WITH_HEADERS.bar" }) assert response.hdrs =~ "foo: bar" assert :ok = Stream.delete(:gnat, "GET_MESSAGE_TEST_WITH_HEADERS") end end describe "purge/2" do test "clears the stream" do stream = %Stream{name: "PURGE_TEST", subjects: ["PURGE_TEST.foo"]} assert {:ok, _response} = Stream.create(:gnat, stream) assert {:ok, _} = Gnat.request(:gnat, "PURGE_TEST.foo", "hi there") assert :ok = Stream.purge(:gnat, "PURGE_TEST") assert {:error, %{"description" => description}} = Stream.get_message(:gnat, "PURGE_TEST", %{ last_by_subj: "PURGE_TEST.foo" }) assert description in ["no message found", "stream store EOF"] assert :ok = Stream.delete(:gnat, "PURGE_TEST") end end end ================================================ FILE: test/jetstream/message_test.exs ================================================ defmodule Gnat.Jetstream.MessageTest do use ExUnit.Case, async: true test "message metadata without domain" do assert {:ok, %Gnat.Jetstream.API.Message.Metadata{ stream_seq: 1_428_472, consumer_seq: 20, num_pending: 3283, num_delivered: 4, timestamp: ~U[2025-06-26 09:09:08.439739Z], stream: "STREAM", consumer: "consumer", domain: nil }} = Gnat.Jetstream.API.Message.metadata(%{ reply_to: "$JS.ACK.STREAM.consumer.4.1428472.20.1750928948439739269.3283" }) end test "message metadata with domain" do assert {:ok, %Gnat.Jetstream.API.Message.Metadata{ stream_seq: 1_428_472, consumer_seq: 20, num_pending: 3283, num_delivered: 4, timestamp: ~U[2025-06-26 09:09:08.439739Z], stream: "STREAM", consumer: "consumer", domain: "test" }} = Gnat.Jetstream.API.Message.metadata(%{ reply_to: "$JS.ACK.test.$G.STREAM.consumer.4.1428472.20.1750928948439739269.3283.279619330" }) end end ================================================ FILE: test/jetstream/pager_test.exs ================================================ defmodule Gnat.Jetstream.PagerTest do use Gnat.Jetstream.ConnCase alias Gnat.Jetstream.Pager alias Gnat.Jetstream.API.Stream @moduletag with_gnat: :gnat test "paging over an entire stream vs from a sequence" do {:ok, _stream} = create_stream("pager_a") Enum.each(1..100, fn i -> :ok = Gnat.pub(:gnat, "input.pager_a", "#{i}") end) {:ok, res} = Pager.reduce(:gnat, "pager_a", [], 0, fn msg, total -> total + String.to_integer(msg.body) end) assert res == 5050 {:ok, res} = Pager.reduce(:gnat, "pager_a", [from_seq: 51], 0, fn msg, total -> total + String.to_integer(msg.body) end) assert res == 3775 Stream.delete(:gnat, "pager_a") end test "paging from a datetime" do {:ok, _stream} = create_stream("pager_b") Enum.each(1..50, fn i -> :ok = Gnat.pub(:gnat, "input.pager_b", "#{i}") end) timestamp = DateTime.utc_now() Enum.each(51..100, fn i -> :ok = Gnat.pub(:gnat, "input.pager_b", "#{i}") end) {:ok, res} = Pager.reduce(:gnat, "pager_b", [from_datetime: timestamp], 0, fn msg, total -> total + String.to_integer(msg.body) end) # The datetime isn't exactly synced between nats and the client # so we use a pretty fuzzy check here. This test is mostly to # provide coverage for accepting the option assert res <= 5050 assert res >= 3775 end defp create_stream(name) do stream = %Stream{ name: name, subjects: ["input.#{name}"] } Stream.create(:gnat, stream) end end ================================================ FILE: test/pull_consumer/batch_test.exs ================================================ defmodule Gnat.Jetstream.PullConsumer.BatchTest do use Gnat.Jetstream.ConnCase alias Gnat.Jetstream.API.{Consumer, Stream} @stream_name "BATCH_TEST_STREAM" @subject "batch_test.*" defmodule BatchPullConsumer do use Gnat.Jetstream.PullConsumer def start_link(opts) do Gnat.Jetstream.PullConsumer.start_link(__MODULE__, opts) end @impl true def init(opts) do batch_size = Keyword.fetch!(opts, :batch_size) connection_opts = [connection_name: :gnat, batch_size: batch_size, request_expires: 500_000_000] |> maybe_put(:consumer, opts) |> maybe_put(:stream_name, opts) |> maybe_put(:consumer_name, opts) |> maybe_put(:connection_retry_timeout, opts) |> maybe_put(:connection_retries, opts) state = %{ test_pid: Keyword.fetch!(opts, :test_pid), messages: [], call_count: 0 } {:ok, state, connection_opts} end defp maybe_put(conn_opts, key, opts) do case Keyword.fetch(opts, key) do {:ok, value} -> Keyword.put(conn_opts, key, value) :error -> conn_opts end end @impl true def handle_connected(consumer_info, state) do send(state.test_pid, {:connected, consumer_info}) {:ok, state} end @impl true def handle_message(message, state) do call_count = state.call_count + 1 messages = [message.body | state.messages] send(state.test_pid, {:handled, call_count, message.body}) {:ack, %{state | messages: messages, call_count: call_count}} end end # A consumer whose handle_message returns :nack or :term based on message body defmodule NonAckBatchConsumer do use Gnat.Jetstream.PullConsumer def start_link(opts) do Gnat.Jetstream.PullConsumer.start_link(__MODULE__, opts) end @impl true def init(opts) do consumer = Keyword.fetch!(opts, :consumer) batch_size = Keyword.fetch!(opts, :batch_size) connection_opts = [ connection_name: :gnat, consumer: consumer, batch_size: batch_size ] state = %{test_pid: Keyword.fetch!(opts, :test_pid)} {:ok, state, connection_opts} end @impl true def handle_message(%{body: "nack_me"}, state) do send(state.test_pid, {:returned_nack}) {:nack, state} end def handle_message(%{body: "term_me"}, state) do send(state.test_pid, {:returned_term}) {:term, state} end def handle_message(message, state) do send(state.test_pid, {:handled, message.body}) {:ack, state} end end describe "batch mode" do @describetag with_gnat: :gnat setup do stream = %Stream{name: @stream_name, subjects: [@subject]} {:ok, _} = Stream.create(:gnat, stream) on_exit(fn -> {:ok, pid} = Gnat.start_link() Stream.delete(pid, @stream_name) Gnat.stop(pid) end) :ok end test "processes a full batch of messages" do consumer = %Consumer{ stream_name: @stream_name, ack_policy: :all, deliver_policy: :all } # Publish exactly batch_size messages before starting consumer for i <- 1..3 do :ok = Gnat.pub(:gnat, "batch_test.full", "msg-#{i}") end start_supervised!({BatchPullConsumer, consumer: consumer, batch_size: 3, test_pid: self()}) assert_receive {:connected, _consumer_info} # All 3 messages should be delivered individually to handle_message assert_receive {:handled, 1, "msg-1"} assert_receive {:handled, 2, "msg-2"} assert_receive {:handled, 3, "msg-3"} end test "processes partial batch when fewer messages than batch_size" do consumer = %Consumer{ stream_name: @stream_name, ack_policy: :all, deliver_policy: :all } # Only 2 messages with batch_size of 5 :ok = Gnat.pub(:gnat, "batch_test.partial", "partial-1") :ok = Gnat.pub(:gnat, "batch_test.partial", "partial-2") start_supervised!({BatchPullConsumer, consumer: consumer, batch_size: 5, test_pid: self()}) assert_receive {:connected, _} # Partial batch should still be processed when terminal signal arrives assert_receive {:handled, 1, "partial-1"}, 3_000 assert_receive {:handled, 2, "partial-2"}, 3_000 end test "empty stream does not hang the consumer" do consumer = %Consumer{ stream_name: @stream_name, ack_policy: :all, deliver_policy: :all } pid = start_supervised!( {BatchPullConsumer, consumer: consumer, batch_size: 5, test_pid: self()} ) assert_receive {:connected, consumer_info} assert consumer_info.num_pending == 0 # Consumer should not hang — it should be alive and responsive assert Process.alive?(pid) # Now publish a message — consumer should still pick it up :ok = Gnat.pub(:gnat, "batch_test.empty", "late-arrival") # In batch mode with batch_size 5, a single message will arrive as a # partial batch (terminal signal triggers processing of the 1-message buffer) assert_receive {:handled, 1, "late-arrival"}, 10_000 end test "continues processing after multiple batches" do consumer = %Consumer{ stream_name: @stream_name, ack_policy: :all, deliver_policy: :all } # Publish 6 messages with batch_size 3 — should produce 2 full batches for i <- 1..6 do :ok = Gnat.pub(:gnat, "batch_test.multi", "multi-#{i}") end start_supervised!({BatchPullConsumer, consumer: consumer, batch_size: 3, test_pid: self()}) assert_receive {:connected, _} # All 6 messages processed across 2 batches for i <- 1..6 do assert_receive {:handled, ^i, _body}, 5_000 end end test "handles messages arriving after initial catch-up" do consumer = %Consumer{ stream_name: @stream_name, ack_policy: :all, deliver_policy: :all } # Start with some messages to catch up on for i <- 1..3 do :ok = Gnat.pub(:gnat, "batch_test.live", "catchup-#{i}") end start_supervised!({BatchPullConsumer, consumer: consumer, batch_size: 3, test_pid: self()}) assert_receive {:connected, _} # Wait for catch-up batch for i <- 1..3 do assert_receive {:handled, ^i, _body}, 5_000 end # Now publish new messages — consumer should transition to tailing mode # and still pick these up :ok = Gnat.pub(:gnat, "batch_test.live", "live-1") :ok = Gnat.pub(:gnat, "batch_test.live", "live-2") assert_receive {:handled, 4, "live-1"}, 10_000 assert_receive {:handled, 5, "live-2"}, 10_000 end test "nack and term returns are treated as ack with warning logged" do consumer = %Consumer{ stream_name: @stream_name, ack_policy: :all, deliver_policy: :all } :ok = Gnat.pub(:gnat, "batch_test.nonack", "normal") :ok = Gnat.pub(:gnat, "batch_test.nonack", "nack_me") :ok = Gnat.pub(:gnat, "batch_test.nonack", "term_me") import ExUnit.CaptureLog log = capture_log(fn -> start_supervised!( {NonAckBatchConsumer, consumer: consumer, batch_size: 3, test_pid: self()} ) assert_receive {:handled, "normal"}, 5_000 assert_receive {:returned_nack}, 5_000 assert_receive {:returned_term}, 5_000 # Give time for the log to flush Process.sleep(100) end) # The warning should mention that batch mode doesn't support nack/term assert log =~ "batch mode does not support" end test "consumer does not get stuck after processing batch" do # This test verifies the critical flow: after a batch is processed and acked, # the consumer must issue another fetch request. If it doesn't, messages # published after the batch will never arrive. consumer = %Consumer{ stream_name: @stream_name, ack_policy: :all, deliver_policy: :all } start_supervised!({BatchPullConsumer, consumer: consumer, batch_size: 2, test_pid: self()}) assert_receive {:connected, _} # First batch :ok = Gnat.pub(:gnat, "batch_test.stuck", "a") :ok = Gnat.pub(:gnat, "batch_test.stuck", "b") assert_receive {:handled, 1, "a"}, 10_000 assert_receive {:handled, 2, "b"}, 10_000 # Wait a moment, then send another batch — consumer must not be stuck Process.sleep(200) :ok = Gnat.pub(:gnat, "batch_test.stuck", "c") :ok = Gnat.pub(:gnat, "batch_test.stuck", "d") assert_receive {:handled, 3, "c"}, 10_000 assert_receive {:handled, 4, "d"}, 10_000 # Third batch to confirm sustained flow Process.sleep(200) :ok = Gnat.pub(:gnat, "batch_test.stuck", "e") assert_receive {:handled, 5, "e"}, 10_000 end test "batch_size 1 uses single-message mode (sends +NXT on ack)" do consumer_name = "BATCH_COMPAT_CONSUMER" consumer = %Consumer{ stream_name: @stream_name, durable_name: consumer_name, inactive_threshold: 30_000_000_000, ack_policy: :explicit } # Subscribe to the ack subject to verify +NXT is sent (single-message pipeline) # rather than an empty body (batch-mode ack). {:ok, _} = Gnat.sub(:gnat, self(), "$JS.ACK.#{@stream_name}.#{consumer_name}.>") :ok = Gnat.pub(:gnat, "batch_test.compat", "compat-1") start_supervised!({BatchPullConsumer, consumer: consumer, batch_size: 1, test_pid: self()}) assert_receive {:handled, 1, "compat-1"}, 5_000 assert_receive {:msg, %{body: "+NXT"}}, 5_000 end test "batch mode with batch_size 2 and odd number of messages" do consumer = %Consumer{ stream_name: @stream_name, ack_policy: :all, deliver_policy: :all } # 5 messages with batch_size 2: batches of [2, 2, 1(partial)] for i <- 1..5 do :ok = Gnat.pub(:gnat, "batch_test.odd", "odd-#{i}") end start_supervised!({BatchPullConsumer, consumer: consumer, batch_size: 2, test_pid: self()}) assert_receive {:connected, _} for i <- 1..5 do assert_receive {:handled, ^i, _body}, 5_000 end end test "large batch processes many messages correctly" do consumer = %Consumer{ stream_name: @stream_name, ack_policy: :all, deliver_policy: :all } msg_count = 50 for i <- 1..msg_count do :ok = Gnat.pub(:gnat, "batch_test.large", "large-#{i}") end start_supervised!({BatchPullConsumer, consumer: consumer, batch_size: 10, test_pid: self()}) assert_receive {:connected, _} # All 50 messages should be delivered (5 batches of 10) for i <- 1..msg_count do assert_receive {:handled, ^i, _body}, 10_000 end end test "can be closed cleanly during batch mode" do consumer = %Consumer{ stream_name: @stream_name, ack_policy: :all, deliver_policy: :all } pid = start_supervised!( {BatchPullConsumer, consumer: consumer, batch_size: 5, test_pid: self()} ) assert_receive {:connected, _} ref = Process.monitor(pid) assert :ok = Gnat.Jetstream.PullConsumer.close(pid) assert_receive {:DOWN, ^ref, :process, ^pid, :shutdown} end test "handle_connected receives consumer info in batch mode" do # Publish messages before consumer starts, so num_pending > 0 for i <- 1..3 do {:ok, _} = Gnat.request(:gnat, "batch_test.connected", "pre-#{i}") end consumer = %Consumer{ stream_name: @stream_name, ack_policy: :all, deliver_policy: :all } start_supervised!({BatchPullConsumer, consumer: consumer, batch_size: 5, test_pid: self()}) assert_receive {:connected, consumer_info} assert consumer_info.num_pending == 3 end test "rejects ephemeral consumer with ack_policy :explicit and batch_size > 1" do consumer = %Consumer{ stream_name: @stream_name, ack_policy: :explicit, deliver_policy: :all } Process.flag(:trap_exit, true) assert {:error, {%ArgumentError{message: message}, _}} = BatchPullConsumer.start_link(consumer: consumer, batch_size: 3, test_pid: self()) assert message =~ "batch_size > 1 requires ack_policy: :all" end test "rejects ephemeral consumer with default ack_policy and batch_size > 1" do # Consumer defaults to ack_policy: :explicit, so forgetting to set it should fail consumer = %Consumer{ stream_name: @stream_name, deliver_policy: :all } Process.flag(:trap_exit, true) assert {:error, {%ArgumentError{message: message}, _}} = BatchPullConsumer.start_link(consumer: consumer, batch_size: 3, test_pid: self()) assert message =~ "batch_size > 1 requires ack_policy: :all" end test "durable consumer with ack_policy :explicit fails to connect in batch mode" do # Create a durable consumer with explicit ack policy consumer = %Consumer{ stream_name: @stream_name, durable_name: "BATCH_EXPLICIT_CONSUMER", ack_policy: :explicit } {:ok, _} = Consumer.create(:gnat, consumer) pid = start_supervised!( {BatchPullConsumer, [ stream_name: @stream_name, consumer_name: "BATCH_EXPLICIT_CONSUMER", batch_size: 3, test_pid: self(), connection_retry_timeout: 50, connection_retries: 1 ]}, restart: :temporary ) ref = Process.monitor(pid) # Should fail to connect and eventually stop after retries assert_receive {:DOWN, ^ref, :process, ^pid, :timeout}, 5_000 end test "allows batch_size 1 with any ack_policy (no validation needed)" do consumer = %Consumer{ stream_name: @stream_name, ack_policy: :explicit, deliver_policy: :all } # batch_size: 1 should not trigger the ack_policy validation pid = start_supervised!( {BatchPullConsumer, consumer: consumer, batch_size: 1, test_pid: self()} ) assert_receive {:connected, _} assert Process.alive?(pid) end end end ================================================ FILE: test/pull_consumer/connectivity_test.exs ================================================ defmodule Gnat.Jetstream.PullConsumer.ConnectivityTest do use Gnat.Jetstream.ConnCase alias Gnat.Jetstream.API.{Consumer, Stream} defmodule ExamplePullConsumer do use Gnat.Jetstream.PullConsumer def start_link(opts) do Gnat.Jetstream.PullConsumer.start_link(__MODULE__, opts) end @impl true def init(opts) do {:ok, nil, Keyword.merge([connection_name: :gnat], opts)} end @impl true def handle_message(%{topic: "ackable"}, state) do {:ack, state} end def handle_message(%{topic: "non-ackable", reply_to: reply_to}, state) do [_, _, _, _, delivered_count, _, _, _, _] = String.split(reply_to, ".") # NACK on first delivery if delivered_count == "1" do {:nack, state} else {:ack, state} end end def handle_message(%{topic: "terminatable"}, state) do {:term, state} end def handle_message(%{topic: "skippable"}, state) do {:noreply, state} end end describe "Jetstream.PullConsumer" do @describetag with_gnat: :gnat setup do stream_name = "TEST_STREAM" stream_subjects = ["ackable", "non-ackable", "terminatable", "skippable"] consumer_name = "TEST_CONSUMER" stream = %Stream{name: stream_name, subjects: stream_subjects} {:ok, _response} = Stream.create(:gnat, stream) consumer = %Consumer{stream_name: stream_name, durable_name: consumer_name} {:ok, _response} = Consumer.create(:gnat, consumer) on_exit(fn -> cleanup() end) %{ stream_name: stream_name, consumer_name: consumer_name } end test "ignores messages with :noreply", %{ stream_name: stream_name, consumer_name: consumer_name } do start_supervised!( {ExamplePullConsumer, stream_name: stream_name, consumer_name: consumer_name} ) :ok = Gnat.pub(:gnat, "skippable", "hello") refute_receive {:msg, _} end test "consumes JetStream messages", %{stream_name: stream_name, consumer_name: consumer_name} do start_supervised!( {ExamplePullConsumer, stream_name: stream_name, consumer_name: consumer_name} ) Gnat.sub(:gnat, self(), "$JS.ACK.#{stream_name}.#{consumer_name}.>") :ok = Gnat.pub(:gnat, "ackable", "hello") assert_receive {:msg, %{body: "+NXT", topic: topic}} assert String.starts_with?(topic, "$JS.ACK.#{stream_name}.#{consumer_name}.1") :ok = Gnat.pub(:gnat, "ackable", "hello") assert_receive {:msg, %{body: "+NXT", topic: topic}} assert String.starts_with?(topic, "$JS.ACK.#{stream_name}.#{consumer_name}.1") :ok = Gnat.pub(:gnat, "non-ackable", "hello") assert_receive {:msg, %{body: "-NAK", topic: topic}} assert String.starts_with?(topic, "$JS.ACK.#{stream_name}.#{consumer_name}.1") assert_receive {:msg, %{body: "+NXT", topic: topic}} assert String.starts_with?(topic, "$JS.ACK.#{stream_name}.#{consumer_name}.2") :ok = Gnat.pub(:gnat, "ackable", "hello") assert_receive {:msg, %{body: "+NXT", topic: topic}} assert String.starts_with?(topic, "$JS.ACK.#{stream_name}.#{consumer_name}.1") :ok = Gnat.pub(:gnat, "terminatable", "hello") assert_receive {:msg, %{body: "+TERM", topic: topic}} assert String.starts_with?(topic, "$JS.ACK.#{stream_name}.#{consumer_name}.1") end test "can be manually closed", %{stream_name: stream_name, consumer_name: consumer_name} do pid = start_supervised!( {ExamplePullConsumer, stream_name: stream_name, consumer_name: consumer_name} ) ref = Process.monitor(pid) assert :ok = Gnat.Jetstream.PullConsumer.close(pid) assert_receive {:DOWN, ^ref, :process, ^pid, :shutdown} end test "retries on unsuccessful connection", %{ stream_name: stream_name, consumer_name: consumer_name } do pid = start_supervised!( {ExamplePullConsumer, connection_name: :non_existent, stream_name: stream_name, consumer_name: consumer_name, connection_retry_timeout: 50, connection_retries: 2}, restart: :temporary ) ref = Process.monitor(pid) assert_receive {:DOWN, ^ref, :process, ^pid, :timeout}, 1_000 end test "allows setting custom inbox prefix", %{ stream_name: stream_name, consumer_name: consumer_name } do start_supervised!( {ExamplePullConsumer, inbox_prefix: "CUSTOM_PREFIX.", stream_name: stream_name, consumer_name: consumer_name} ) Gnat.sub(:gnat, self(), "$JS.API.CONSUMER.MSG.NEXT.#{stream_name}.#{consumer_name}") :ok = Gnat.pub(:gnat, "ackable", "hello") expected_body = %{batch: 1} |> Jason.encode!() assert_receive {:msg, %{body: ^expected_body, reply_to: "CUSTOM_PREFIX." <> _}} end end defp cleanup do # Manage connection on our own here, because all supervised processes will be # closed by the time `on_exit` runs {:ok, pid} = Gnat.start_link() :ok = Stream.delete(pid, "TEST_STREAM") Gnat.stop(pid) end end ================================================ FILE: test/pull_consumer/ephemeral_test.exs ================================================ defmodule Gnat.Jetstream.PullConsumer.EphemeralTest do use Gnat.Jetstream.ConnCase alias Gnat.Jetstream.API.{Consumer, Stream} defmodule ExamplePullConsumer do use Gnat.Jetstream.PullConsumer def start_link(opts) do Gnat.Jetstream.PullConsumer.start_link(__MODULE__, opts) end @impl true def init(opts) do consumer = Keyword.fetch!(opts, :consumer) connection_opts = [ connection_name: :gnat, consumer: consumer ] state = %{test_pid: Keyword.fetch!(opts, :test_pid)} {:ok, state, connection_opts} end @impl true def handle_connected(consumer_info, state) do send(state.test_pid, {:connected, consumer_info}) {:ok, state} end @impl true def handle_message(message, state) do send(state.test_pid, {:pulled, message}) {:ack, state} end end describe "Jetstream.PullConsumer" do @describetag with_gnat: :gnat setup do stream_name = "TEST_STREAM_2" stream_subjects = ["stream_2.*"] stream = %Stream{name: stream_name, subjects: stream_subjects} {:ok, _response} = Stream.create(:gnat, stream) on_exit(fn -> cleanup() end) %{ stream_name: stream_name } end test "ephemeral consumer receives messages", %{ stream_name: stream_name } do consumer = %Consumer{stream_name: stream_name} {:ok, _resp} = Gnat.request(:gnat, "stream_2.ohai", "whatsup") start_supervised!({ExamplePullConsumer, consumer: consumer, test_pid: self()}) assert_receive {:connected, consumer_info} assert consumer_info.num_pending == 1 assert_receive {:pulled, message} assert %{topic: "stream_2.ohai", body: "whatsup"} = message {:ok, _resp} = Gnat.request(:gnat, "stream_2.ohai", "second") assert_receive {:pulled, message} assert %{topic: "stream_2.ohai", body: "second"} = message end end defp cleanup do # Manage connection on our own here, because all supervised processes will be # closed by the time `on_exit` runs {:ok, pid} = Gnat.start_link() :ok = Stream.delete(pid, "TEST_STREAM_2") Gnat.stop(pid) end end ================================================ FILE: test/pull_consumer/status_messages_test.exs ================================================ defmodule Gnat.Jetstream.PullConsumer.StatusMessagesTest do use Gnat.Jetstream.ConnCase alias Gnat.Jetstream.API.{Consumer, Stream} defmodule ObservingConsumer do use Gnat.Jetstream.PullConsumer def start_link(opts) do Gnat.Jetstream.PullConsumer.start_link(__MODULE__, opts) end @impl true def init(opts) do {test_pid, opts} = Keyword.pop!(opts, :test_pid) state = Keyword.merge([connection_name: :gnat], opts) {:ok, %{test_pid: test_pid}, state} end @impl true def handle_message(message, state) do send(state.test_pid, {:handle_message, message}) {:ack, state} end @impl true def handle_status(message, state) do send(state.test_pid, {:handle_status, message}) {:ok, state} end end defmodule SilentConsumer do use Gnat.Jetstream.PullConsumer def start_link(opts) do Gnat.Jetstream.PullConsumer.start_link(__MODULE__, opts) end @impl true def init(opts) do {test_pid, opts} = Keyword.pop!(opts, :test_pid) {:ok, test_pid, Keyword.merge([connection_name: :gnat], opts)} end @impl true def handle_message(message, test_pid) do send(test_pid, {:handle_message, message}) {:ack, test_pid} end end describe "JetStream status messages" do @describetag with_gnat: :gnat setup do stream_name = "STATUS_TEST_STREAM" consumer_name = "STATUS_TEST_CONSUMER" stream = %Stream{name: stream_name, subjects: ["status.test"]} {:ok, _} = Stream.create(:gnat, stream) consumer = %Consumer{stream_name: stream_name, durable_name: consumer_name} {:ok, _} = Consumer.create(:gnat, consumer) on_exit(fn -> {:ok, pid} = Gnat.start_link() :ok = Stream.delete(pid, stream_name) Gnat.stop(pid) end) %{stream_name: stream_name, consumer_name: consumer_name} end test "are not forwarded to handle_message when no handle_status is defined", %{stream_name: stream_name, consumer_name: consumer_name} do pid = start_supervised!( {SilentConsumer, test_pid: self(), stream_name: stream_name, consumer_name: consumer_name} ) send(pid, {:msg, %{status: "409", description: "Leadership Change", body: "", gnat: :gnat}}) send(pid, {:msg, %{status: "100", body: "", gnat: :gnat}}) refute_receive {:handle_message, _}, 100 end test "are forwarded to handle_status when the callback is defined", %{stream_name: stream_name, consumer_name: consumer_name} do pid = start_supervised!( {ObservingConsumer, test_pid: self(), stream_name: stream_name, consumer_name: consumer_name} ) send(pid, {:msg, %{status: "409", description: "Leadership Change", body: "", gnat: :gnat}}) assert_receive {:handle_status, %{status: "409", description: "Leadership Change"}} refute_receive {:handle_message, _}, 100 end test "single-message mode issues a new pull after a 409", %{stream_name: stream_name, consumer_name: consumer_name} do # Subscribe to the next-message subject to observe outbound pulls. {:ok, _sid} = Gnat.sub(:gnat, self(), "$JS.API.CONSUMER.MSG.NEXT.#{stream_name}.#{consumer_name}") pid = start_supervised!( {SilentConsumer, test_pid: self(), stream_name: stream_name, consumer_name: consumer_name} ) # Drain the initial pull issued on connect. assert_receive {:msg, %{topic: "$JS.API.CONSUMER.MSG.NEXT." <> _}}, 1_000 send(pid, {:msg, %{status: "409", description: "Leadership Change", body: "", gnat: :gnat}}) # A new pull must be issued or the consumer is stuck. assert_receive {:msg, %{topic: "$JS.API.CONSUMER.MSG.NEXT." <> _}}, 1_000 end end describe "batch-mode JetStream status messages" do @describetag with_gnat: :gnat setup do stream_name = "STATUS_BATCH_STREAM" subject = "status.batch" stream = %Stream{name: stream_name, subjects: [subject]} {:ok, _} = Stream.create(:gnat, stream) on_exit(fn -> {:ok, pid} = Gnat.start_link() :ok = Stream.delete(pid, stream_name) Gnat.stop(pid) end) %{stream_name: stream_name, subject: subject} end defmodule BatchObservingConsumer do use Gnat.Jetstream.PullConsumer def start_link(opts) do Gnat.Jetstream.PullConsumer.start_link(__MODULE__, opts) end @impl true def init(opts) do {test_pid, opts} = Keyword.pop!(opts, :test_pid) state = Keyword.merge([connection_name: :gnat], opts) {:ok, %{test_pid: test_pid}, state} end @impl true def handle_message(message, state) do send(state.test_pid, {:handled, message.body}) {:ack, state} end @impl true def handle_status(message, state) do send(state.test_pid, {:status, message}) {:ok, state} end end test "batch mode issues a new pull after 409 with empty buffer", %{stream_name: stream_name} do consumer = %Gnat.Jetstream.API.Consumer{ stream_name: stream_name, ack_policy: :all, deliver_policy: :all } # Subscribe to the next-message subject before starting the consumer so # we see the pull the consumer issues at connect time and any re-pulls. {:ok, _sid} = Gnat.sub(:gnat, self(), "$JS.API.CONSUMER.MSG.NEXT.#{stream_name}.>") pid = start_supervised!( {BatchObservingConsumer, consumer: consumer, batch_size: 5, test_pid: self()} ) # Drain pulls issued during connect (at minimum the initial catch-up fetch). # Absorb any that arrive within a short window so the assert below only # fires on the post-409 re-pull. drain_pulls(200) send(pid, {:msg, %{status: "409", description: "Leadership Change", body: "", gnat: :gnat}}) # A new pull must be issued or the consumer is stuck. assert_receive {:msg, %{topic: "$JS.API.CONSUMER.MSG.NEXT." <> _}}, 1_000 # And handle_status should have observed the 409. assert_receive {:status, %{status: "409", description: "Leadership Change"}}, 1_000 end test "batch mode processes partial buffer and re-pulls on 409", %{stream_name: stream_name, subject: subject} do consumer = %Gnat.Jetstream.API.Consumer{ stream_name: stream_name, ack_policy: :all, deliver_policy: :all } {:ok, _sid} = Gnat.sub(:gnat, self(), "$JS.API.CONSUMER.MSG.NEXT.#{stream_name}.>") # Publish a single message so the consumer buffers 1 of a 5-message batch. :ok = Gnat.pub(:gnat, subject, "buffered-1") pid = start_supervised!( {BatchObservingConsumer, consumer: consumer, batch_size: 5, test_pid: self()} ) # The buffered message should arrive at the server-issued 404 terminator # and be processed. Once that's settled, drain any further pulls. assert_receive {:handled, "buffered-1"}, 5_000 drain_pulls(200) # Now stuff a new message into the buffer via direct send and inject a 409. fake_msg = %{ topic: subject, body: "partial-1", reply_to: "$JS.ACK.#{stream_name}.FAKE.1.1.1.0.0", gnat: :gnat } send(pid, {:msg, fake_msg}) # A 409 should cause the partial buffer to be processed (handle_message # called for "partial-1") and a fresh pull to be issued. send(pid, {:msg, %{status: "409", description: "Leadership Change", body: "", gnat: :gnat}}) assert_receive {:handled, "partial-1"}, 1_000 assert_receive {:msg, %{topic: "$JS.API.CONSUMER.MSG.NEXT." <> _}}, 1_000 assert_receive {:status, %{status: "409", description: "Leadership Change"}}, 1_000 end test "batch mode treats 100 heartbeat as a no-op (no re-pull)", %{stream_name: stream_name} do consumer = %Gnat.Jetstream.API.Consumer{ stream_name: stream_name, ack_policy: :all, deliver_policy: :all } {:ok, _sid} = Gnat.sub(:gnat, self(), "$JS.API.CONSUMER.MSG.NEXT.#{stream_name}.>") pid = start_supervised!( {BatchObservingConsumer, consumer: consumer, batch_size: 5, test_pid: self()} ) drain_pulls(200) # 100 is a keep-alive on the existing pull — must NOT cause a re-pull. send(pid, {:msg, %{status: "100", body: "", gnat: :gnat}}) refute_receive {:msg, %{topic: "$JS.API.CONSUMER.MSG.NEXT." <> _}}, 200 # But handle_status should still be invoked so users can observe heartbeats. assert_receive {:status, %{status: "100"}}, 500 end end defp drain_pulls(timeout) do receive do {:msg, %{topic: "$JS.API.CONSUMER.MSG.NEXT." <> _}} -> drain_pulls(timeout) after timeout -> :ok end end end ================================================ FILE: test/pull_consumer/using_macro_test.exs ================================================ defmodule Gnat.Jetstream.PullConsumer.UsingMacroTest do use Gnat.Jetstream.ConnCase defmodule ExamplePullConsumer do use Gnat.Jetstream.PullConsumer, restart: :temporary, shutdown: 12345 def start_link(_) do Gnat.Jetstream.PullConsumer.start_link(__MODULE__, []) end @impl true def init([]) do {:ok, nil, []} end @impl true def handle_message(%{}, state) do {:ack, state} end end describe "use Jetstream.PullConsumer" do test "allows specifing child specification options via argument" do assert ExamplePullConsumer.child_spec(:arg) == %{ id: ExamplePullConsumer, start: {ExamplePullConsumer, :start_link, [:arg]}, restart: :temporary, shutdown: 12345 } end end end ================================================ FILE: test/support/conn_case.ex ================================================ defmodule Gnat.Jetstream.ConnCase do @moduledoc """ This module defines the test case to be used by tests that require setting up a connection. """ use ExUnit.CaseTemplate using(opts) do tags = module_tags(opts) quote do import Gnat.Jetstream.ConnCase @moduletag unquote(tags) end end defp module_tags(opts) do Enum.reduce(opts, [capture_log: true], fn opt, acc -> add_module_tag(acc, opt) end) end defp add_module_tag(tags, {:min_server_version, min_version}) do if server_version_incompatible?(min_version) do Keyword.put(tags, :incompatible, true) else tags end end defp add_module_tag(tags, _opt), do: tags defp get_server_version(conn) do Gnat.server_info(conn).version end defp server_version_incompatible?(min_version) do {:ok, conn} = Gnat.start_link() match? = Version.match?(get_server_version(conn), ">= #{min_version}") :ok = Gnat.stop(conn) !match? end setup tags do if arg = Map.get(tags, :with_gnat) do conn = start_gnat!(arg) %{conn: conn} else :ok end end def start_gnat!(name) when is_atom(name) do start_gnat!(%{}, name: name) end def start_gnat!(connection_settings, options) when is_map(connection_settings) and is_list(options) do {Gnat, %{}} |> Supervisor.child_spec(start: {Gnat, :start_link, [connection_settings, options]}) |> start_supervised!() end end ================================================ FILE: test/support/generators.ex ================================================ defmodule Gnat.Generators do use PropCheck # Character classes useful for generating text def alphanumeric_char do elements(~c"0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ") end def alphanumeric_space_char do elements(~c" 0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ") end def numeric_char do elements(~c"0123456789") end @doc "protocol delimiter. The protocol can use one ore more of space or tab characters as delimiters between fields" def delimiter, do: let(chunks <- non_empty(list(delimiter_char())), do: Enum.join(chunks, "")) def delimiter_char, do: union([" ", "\t"]) def error do let chars <- list(alphanumeric_space_char()) do %{binary: "-ERR '#{List.to_string(chars)}'\r\n"} end end def host_port do let( {ip, port} <- {[byte(), byte(), byte(), byte()], non_neg_integer()}, do: "#{Enum.join(ip, ".")}:#{port}" ) end def info do let options <- info_options() do %{binary: "INFO #{Jason.encode!(options)}\r\n"} end end def info_options do let( {server_id, version, go, host, port, auth_required, ssl_required, max_payload, connect_urls} <- {list(alphanumeric_space_char()), list(list(numeric_char())), list(alphanumeric_char()), list(alphanumeric_char()), non_neg_integer(), boolean(), boolean(), integer(1024, 1_048_576), list(host_port())} ) do %{ server_id: List.to_string(server_id), version: version |> Enum.map(&List.to_string/1) |> Enum.join("."), go: List.to_string(go), host: List.to_string(host), port: port, auth_required: auth_required, ssl_required: ssl_required, max_payload: max_payload, connect_urls: connect_urls } end end def ok, do: %{binary: "+OK\r\n"} def ping, do: %{binary: "PING\r\n"} def pong, do: %{binary: "PONG\r\n"} # generates a map containing the binary encoded message and attributes for which generated # sid, subject, payload and reply_to topic were used in the encoded message def message, do: sized(size, message(size)) def message(size), do: union([message_without_reply(size), message_with_reply(size)]) def message_with_reply(size) do let( {p, su, si, r, d1, d2, d3, d4} <- {payload(size), subject(), sid(), reply_to(), delimiter(), delimiter(), delimiter(), delimiter()} ) do parts = ["MSG", d1, su, d2, si, d3, r, d4, byte_size(p), "\r\n", p, "\r\n"] %{ binary: Enum.join(parts), reply_to: r, sid: si, subject: su, payload: p } end end def message_without_reply(size) do let( {p, su, si, d1, d2, d3} <- {payload(size), subject(), sid(), delimiter(), delimiter(), delimiter()} ) do parts = ["MSG", d1, su, d2, si, d3, byte_size(p), "\r\n", p, "\r\n"] %{ binary: Enum.join(parts), reply_to: nil, sid: si, subject: su, payload: p } end end def payload(size), do: binary(size) def protocol_message do union([ok(), ping(), pong(), error(), info(), message()]) end # according to the spec sid's can be alphanumeric, but our client only generates # non-negative integers and we only receive back our own sids def sid, do: non_neg_integer() def subject do let(chunks <- subject_chunks(), do: Enum.join(chunks, ".")) end def subject_chunks do non_empty(list(non_empty(list(alphanumeric_char())))) end # TODO subsription names are like subject names, but they can have wildcards # There is a special case where a user can subscribe to ">" to subscribe to all topics def reply_to, do: subject() end ================================================ FILE: test/test_helper.exs ================================================ ExUnit.configure(exclude: [:pending, :property, :multi_server, :message_ttl]) ExUnit.start() # set assert_receive default timeout Application.put_env(:ex_unit, :assert_receive_timeout, 1_000) # cleanup any streams left over by previous test runs {:ok, conn} = Gnat.start_link(%{}, name: :jstest) {:ok, %{streams: streams}} = Gnat.Jetstream.API.Stream.list(conn) streams = streams || [] Enum.each(streams, fn stream -> :ok = Gnat.Jetstream.API.Stream.delete(conn, stream) end) :ok = Gnat.stop(conn) case :gen_tcp.connect(~c"localhost", 4222, [:binary]) do {:ok, socket} -> :gen_tcp.close(socket) {:error, reason} -> Mix.raise( "Cannot connect to nats-server" <> " (http://localhost:4222):" <> " #{:inet.format_error(reason)}\n" <> "You probably need to start nats-server." ) end # this is used by some property tests, see test/gnat_property_test.exs Gnat.start_link(%{}, name: :test_connection) defmodule RpcEndpoint do def init do {:ok, pid} = Gnat.start_link() {:ok, _ref} = Gnat.sub(pid, self(), "rpc.>") loop(pid) end def loop(pid) do receive do {:msg, %{body: body, reply_to: topic}} -> Gnat.pub(pid, topic, body) loop(pid) end end end spawn(&RpcEndpoint.init/0) defmodule ExampleService do use Gnat.Services.Server def request(%{topic: "calc.add", body: body}, _endpoint, _group) when body == "foo" do :timer.sleep(10) {:reply, "6"} end def request(%{body: body}, "sub", "calc") when body == "foo" do # want some processing time to show up :timer.sleep(10) {:reply, "4"} end def request(_, _, _) do {:error, "oops"} end def error(_msg, "oops") do {:reply, "500 error"} end end defmodule ExampleServer do use Gnat.Server def request(%{topic: "example.good", body: body}) do {:reply, "Re: #{body}"} end def request(%{topic: "example.error"}) do {:error, "oops"} end def request(%{topic: "example.raise"}) do raise "oops" end def error(_msg, "oops") do {:reply, "400 error"} end def error(_msg, %RuntimeError{message: "oops"}) do {:reply, "500 error"} end def error(msg, other) do require Logger Logger.error("#{msg.topic} failed #{inspect(other)}") end end {:ok, _pid} = Gnat.ConsumerSupervisor.start_link(%{ connection_name: :test_connection, module: ExampleServer, subscription_topics: [ %{topic: "example.*"} ] }) {:ok, _pid} = Gnat.ConsumerSupervisor.start_link(%{ connection_name: :test_connection, module: ExampleService, service_definition: %{ name: "exampleservice", description: "This is an example service", version: "0.1.0", endpoints: [ %{ name: "add", group_name: "calc" }, %{ name: "sub", group_name: "calc" } ] } }) defmodule CheckForExpectedNatsServers do def check(tags) do check_for_default() Enum.each(tags, &check_for_tag/1) end def check_for_default do case :gen_tcp.connect(~c"localhost", 4222, [:binary]) do {:ok, socket} -> :gen_tcp.close(socket) {:error, reason} -> Mix.raise( "Cannot connect to nats-server" <> " (tcp://localhost:4222):" <> " #{:inet.format_error(reason)}\n" <> "You probably need to start nats-server." ) end end def check_for_tag(:multi_server) do case :gen_tcp.connect(~c"localhost", 4223, [:binary]) do {:ok, socket} -> :gen_tcp.close(socket) {:error, reason} -> Mix.raise( "Cannot connect to nats-server" <> " (tcp://localhost:4223):" <> " #{:inet.format_error(reason)}\n" <> "You probably need to start a nats-server " <> "server that requires authentication with " <> "the following command `nats-server -p 4223 " <> "--user bob --pass alice`." ) end case :gen_tcp.connect(~c"localhost", 4224, [:binary]) do {:ok, socket} -> :gen_tcp.close(socket) {:error, reason} -> Mix.raise( "Cannot connect to nats-server" <> " (tcp://localhost:4224):" <> " #{:inet.format_error(reason)}\n" <> "You probably need to start a nats-server " <> "server that requires tls with " <> "a command like `nats-server -p 4224 " <> "--tls --tlscert test/fixtures/server-cert.pem " <> "--tlskey test/fixtures/server-key.pem`." ) end case :gen_tcp.connect(~c"localhost", 4225, [:binary]) do {:ok, socket} -> :gen_tcp.close(socket) {:error, reason} -> Mix.raise( "Cannot connect to nats-server" <> " (tcp://localhost:4225):" <> " #{:inet.format_error(reason)}\n" <> "You probably need to start a nats-server " <> "server that requires tls with " <> "a command like `nats-server -p 4225 --tls " <> "--tlscert test/fixtures/server-cert.pem " <> "--tlskey test/fixtures/server-key.pem " <> "--tlscacert test/fixtures/ca.pem --tlsverify" ) end case :gen_tcp.connect(~c"localhost", 4226, [:binary]) do {:ok, socket} -> :gen_tcp.close(socket) {:error, reason} -> Mix.raise( "Cannot connect to nats-server" <> " (tcp://localhost:4226):" <> " #{:inet.format_error(reason)}\n" <> "You probably need to start a nats-server " <> "server that requires authentication with " <> "the following command `nats-server -p 4226 " <> "--auth SpecialToken`." ) end case :gen_tcp.connect(~c"localhost", 4227, [:binary]) do {:ok, socket} -> :gen_tcp.close(socket) {:error, reason} -> Mix.raise( "Cannot connect to nats-server" <> " (tcp://localhost:4227):" <> " #{:inet.format_error(reason)}\n" <> "You probably need to start a nats-server " <> "server that requires authentication with " <> "the following command `nats-server -p 4227 " <> "-c test/fixtures/nkey_config`." ) end end def check_for_tag(_), do: :ok end